@cryptotaxi247 / kubo / commits / c3692a591

extract the pinner to go-ipfs-pinner and dagutils into go-merkledag

Michael Muré committed Nov 20, 2019 at 16:38 UTC c3692a5913b655af24f1cf875eb3cc8f9f303124
33 files changed +73 -2827
Rules.mk
-3
@@ -48,9 +48,6 @@ ifneq ($(filter coverage% clean distclean test/unit/gotest.junit.xml,$(MAKECMDGO
48 include $(dir)/Rules.mk
49 endif
50
51 -dir := pin/internal/pb
52 -include $(dir)/Rules.mk
53 -
51 # -------------------- #
52 # universal rules #
53 # -------------------- #
blocks/blockstoreutil/remove.go
+1 -2
@@ -6,10 +6,9 @@ import (
6 "fmt"
7 "io"
8
9 - "github.com/ipfs/go-ipfs/pin"
10 -
9 cid "github.com/ipfs/go-cid"
10 bs "github.com/ipfs/go-ipfs-blockstore"
11 + "github.com/ipfs/go-ipfs-pinner"
12 )
13
14 // RemovedBlock is used to respresent the result of removing a block.
core/commands/object/diff.go
+4 -4
@@ -4,11 +4,11 @@ import (
4 "fmt"
5 "io"
6
7 - cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
8 - "github.com/ipfs/go-ipfs/dagutils"
9 -
7 cmds "github.com/ipfs/go-ipfs-cmds"
8 + "github.com/ipfs/go-merkledag/dagutils"
9 path "github.com/ipfs/interface-go-ipfs-core/path"
10 +
11 + cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
12 )
13
14 const (
@@ -70,7 +70,7 @@ Example:
70 out := make([]*dagutils.Change, len(changes))
71 for i, change := range changes {
72 out[i] = &dagutils.Change{
73 - Type: change.Type,
73 + Type: dagutils.ChangeType(change.Type),
74 Path: change.Path,
75 }
76
core/commands/pin.go
+6 -6
@@ -8,23 +8,23 @@ import (
8 "os"
9 "time"
10
11 - core "github.com/ipfs/go-ipfs/core"
12 - cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
13 - e "github.com/ipfs/go-ipfs/core/commands/e"
14 - coreapi "github.com/ipfs/go-ipfs/core/coreapi"
15 - pin "github.com/ipfs/go-ipfs/pin"
16 -
11 bserv "github.com/ipfs/go-blockservice"
12 cid "github.com/ipfs/go-cid"
13 cidenc "github.com/ipfs/go-cidutil/cidenc"
14 cmds "github.com/ipfs/go-ipfs-cmds"
15 offline "github.com/ipfs/go-ipfs-exchange-offline"
16 + pin "github.com/ipfs/go-ipfs-pinner"
17 ipld "github.com/ipfs/go-ipld-format"
18 dag "github.com/ipfs/go-merkledag"
19 verifcid "github.com/ipfs/go-verifcid"
20 coreiface "github.com/ipfs/interface-go-ipfs-core"
21 options "github.com/ipfs/interface-go-ipfs-core/options"
22 "github.com/ipfs/interface-go-ipfs-core/path"
23 +
24 + core "github.com/ipfs/go-ipfs/core"
25 + cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
26 + e "github.com/ipfs/go-ipfs/core/commands/e"
27 + coreapi "github.com/ipfs/go-ipfs/core/coreapi"
28 )
29
30 var PinCmd = &cmds.Command{
core/core.go
+10 -9
@@ -14,15 +14,7 @@ import (
14 "io"
15
16 "github.com/ipfs/go-filestore"
17 - "github.com/ipfs/go-ipfs/core/bootstrap"
18 - "github.com/ipfs/go-ipfs/core/node"
19 - "github.com/ipfs/go-ipfs/core/node/libp2p"
20 - "github.com/ipfs/go-ipfs/fuse/mount"
21 - "github.com/ipfs/go-ipfs/namesys"
22 - ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher"
23 - "github.com/ipfs/go-ipfs/p2p"
24 - "github.com/ipfs/go-ipfs/pin"
25 - "github.com/ipfs/go-ipfs/repo"
17 + "github.com/ipfs/go-ipfs-pinner"
18
19 bserv "github.com/ipfs/go-blockservice"
20 bstore "github.com/ipfs/go-ipfs-blockstore"
@@ -47,6 +39,15 @@ import (
39 record "github.com/libp2p/go-libp2p-record"
40 "github.com/libp2p/go-libp2p/p2p/discovery"
41 p2pbhost "github.com/libp2p/go-libp2p/p2p/host/basic"
42 +
43 + "github.com/ipfs/go-ipfs/core/bootstrap"
44 + "github.com/ipfs/go-ipfs/core/node"
45 + "github.com/ipfs/go-ipfs/core/node/libp2p"
46 + "github.com/ipfs/go-ipfs/fuse/mount"
47 + "github.com/ipfs/go-ipfs/namesys"
48 + ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher"
49 + "github.com/ipfs/go-ipfs/p2p"
50 + "github.com/ipfs/go-ipfs/repo"
51 )
52
53 var log = logging.Logger("core")
core/coreapi/block.go
+3 -3
@@ -7,14 +7,14 @@ import (
7 "io"
8 "io/ioutil"
9
10 - util "github.com/ipfs/go-ipfs/blocks/blockstoreutil"
11 - pin "github.com/ipfs/go-ipfs/pin"
12 -
10 blocks "github.com/ipfs/go-block-format"
11 cid "github.com/ipfs/go-cid"
12 + pin "github.com/ipfs/go-ipfs-pinner"
13 coreiface "github.com/ipfs/interface-go-ipfs-core"
14 caopts "github.com/ipfs/interface-go-ipfs-core/options"
15 path "github.com/ipfs/interface-go-ipfs-core/path"
16 +
17 + util "github.com/ipfs/go-ipfs/blocks/blockstoreutil"
18 )
19
20 type BlockAPI CoreAPI
core/coreapi/coreapi.go
+6 -6
@@ -18,16 +18,11 @@ import (
18 "errors"
19 "fmt"
20
21 - "github.com/ipfs/go-ipfs/core"
22 - "github.com/ipfs/go-ipfs/core/node"
23 - "github.com/ipfs/go-ipfs/namesys"
24 - "github.com/ipfs/go-ipfs/pin"
25 - "github.com/ipfs/go-ipfs/repo"
26 -
21 bserv "github.com/ipfs/go-blockservice"
22 "github.com/ipfs/go-ipfs-blockstore"
23 "github.com/ipfs/go-ipfs-exchange-interface"
24 offlinexch "github.com/ipfs/go-ipfs-exchange-offline"
25 + "github.com/ipfs/go-ipfs-pinner"
26 "github.com/ipfs/go-ipfs-provider"
27 offlineroute "github.com/ipfs/go-ipfs-routing/offline"
28 ipld "github.com/ipfs/go-ipld-format"
@@ -42,6 +37,11 @@ import (
37 routing "github.com/libp2p/go-libp2p-core/routing"
38 pubsub "github.com/libp2p/go-libp2p-pubsub"
39 record "github.com/libp2p/go-libp2p-record"
40 +
41 + "github.com/ipfs/go-ipfs/core"
42 + "github.com/ipfs/go-ipfs/core/node"
43 + "github.com/ipfs/go-ipfs/namesys"
44 + "github.com/ipfs/go-ipfs/repo"
45 )
46
47 var log = logging.Logger("core/coreapi")
core/coreapi/dag.go
+1 -2
@@ -3,9 +3,8 @@ package coreapi
3 import (
4 "context"
5
6 - "github.com/ipfs/go-ipfs/pin"
7 -
6 cid "github.com/ipfs/go-cid"
7 + "github.com/ipfs/go-ipfs-pinner"
8 ipld "github.com/ipfs/go-ipld-format"
9 )
10
core/coreapi/object.go
+3 -4
@@ -11,12 +11,11 @@ import (
11 "io"
12 "io/ioutil"
13
14 - "github.com/ipfs/go-ipfs/dagutils"
15 - "github.com/ipfs/go-ipfs/pin"
16 -
14 cid "github.com/ipfs/go-cid"
15 + "github.com/ipfs/go-ipfs-pinner"
16 ipld "github.com/ipfs/go-ipld-format"
17 dag "github.com/ipfs/go-merkledag"
18 + "github.com/ipfs/go-merkledag/dagutils"
19 ft "github.com/ipfs/go-unixfs"
20 coreiface "github.com/ipfs/interface-go-ipfs-core"
21 caopts "github.com/ipfs/interface-go-ipfs-core/options"
@@ -307,7 +306,7 @@ func (api *ObjectAPI) Diff(ctx context.Context, before ipath.Path, after ipath.P
306 out := make([]coreiface.ObjectChange, len(changes))
307 for i, change := range changes {
308 out[i] = coreiface.ObjectChange{
310 - Type: change.Type,
309 + Type: coreiface.ChangeType(change.Type),
310 Path: change.Path,
311 }
312
core/coreapi/pin.go
+1 -1
@@ -6,7 +6,7 @@ import (
6 bserv "github.com/ipfs/go-blockservice"
7 "github.com/ipfs/go-cid"
8 offline "github.com/ipfs/go-ipfs-exchange-offline"
9 - "github.com/ipfs/go-ipfs/pin"
9 + pin "github.com/ipfs/go-ipfs-pinner"
10 ipld "github.com/ipfs/go-ipld-format"
11 "github.com/ipfs/go-merkledag"
12 coreiface "github.com/ipfs/interface-go-ipfs-core"
core/corerepo/gc.go
+5 -5
@@ -7,13 +7,13 @@ import (
7 "time"
8
9 "github.com/ipfs/go-ipfs/core"
10 - gc "github.com/ipfs/go-ipfs/pin/gc"
11 - repo "github.com/ipfs/go-ipfs/repo"
10 + "github.com/ipfs/go-ipfs/gc"
11 + "github.com/ipfs/go-ipfs/repo"
12
13 - humanize "github.com/dustin/go-humanize"
14 - cid "github.com/ipfs/go-cid"
13 + "github.com/dustin/go-humanize"
14 + "github.com/ipfs/go-cid"
15 logging "github.com/ipfs/go-log"
16 - mfs "github.com/ipfs/go-mfs"
16 + "github.com/ipfs/go-mfs"
17 )
18
19 var log = logging.Logger("corerepo")
core/coreunix/add.go
+1 -2
@@ -8,12 +8,11 @@ import (
8 gopath "path"
9 "strconv"
10
11 - "github.com/ipfs/go-ipfs/pin"
12 -
11 "github.com/ipfs/go-cid"
12 bstore "github.com/ipfs/go-ipfs-blockstore"
13 chunker "github.com/ipfs/go-ipfs-chunker"
14 "github.com/ipfs/go-ipfs-files"
15 + "github.com/ipfs/go-ipfs-pinner"
16 "github.com/ipfs/go-ipfs-posinfo"
17 ipld "github.com/ipfs/go-ipld-format"
18 logging "github.com/ipfs/go-log"
core/coreunix/add_test.go
+3 -3
@@ -12,13 +12,13 @@ import (
12 "time"
13
14 "github.com/ipfs/go-ipfs/core"
15 - "github.com/ipfs/go-ipfs/pin/gc"
15 + "github.com/ipfs/go-ipfs/gc"
16 "github.com/ipfs/go-ipfs/repo"
17
18 blocks "github.com/ipfs/go-block-format"
19 "github.com/ipfs/go-blockservice"
20 - cid "github.com/ipfs/go-cid"
21 - datastore "github.com/ipfs/go-datastore"
20 + "github.com/ipfs/go-cid"
21 + "github.com/ipfs/go-datastore"
22 syncds "github.com/ipfs/go-datastore/sync"
23 blockstore "github.com/ipfs/go-ipfs-blockstore"
24 config "github.com/ipfs/go-ipfs-config"
core/node/core.go
+4 -4
@@ -4,10 +4,6 @@ import (
4 "context"
5 "fmt"
6
7 - "github.com/ipfs/go-ipfs/core/node/helpers"
8 - "github.com/ipfs/go-ipfs/pin"
9 - "github.com/ipfs/go-ipfs/repo"
10 -
7 "github.com/ipfs/go-bitswap"
8 "github.com/ipfs/go-bitswap/network"
9 "github.com/ipfs/go-blockservice"
@@ -16,6 +12,7 @@ import (
12 "github.com/ipfs/go-ipfs-blockstore"
13 "github.com/ipfs/go-ipfs-exchange-interface"
14 "github.com/ipfs/go-ipfs-exchange-offline"
15 + "github.com/ipfs/go-ipfs-pinner"
16 "github.com/ipfs/go-ipld-format"
17 "github.com/ipfs/go-merkledag"
18 "github.com/ipfs/go-mfs"
@@ -23,6 +20,9 @@ import (
20 "github.com/libp2p/go-libp2p-core/host"
21 "github.com/libp2p/go-libp2p-core/routing"
22 "go.uber.org/fx"
23 +
24 + "github.com/ipfs/go-ipfs/core/node/helpers"
25 + "github.com/ipfs/go-ipfs/repo"
26 )
27
28 // BlockService creates new blockservice which provides an interface to fetch content-addressable blocks
core/node/provider.go
+4 -4
@@ -5,16 +5,16 @@ import (
5 "fmt"
6 "time"
7
8 - "github.com/ipfs/go-ipfs/core/node/helpers"
9 - "github.com/ipfs/go-ipfs/pin"
10 - "github.com/ipfs/go-ipfs/repo"
11 -
8 + "github.com/ipfs/go-ipfs-pinner"
9 "github.com/ipfs/go-ipfs-provider"
10 q "github.com/ipfs/go-ipfs-provider/queue"
11 "github.com/ipfs/go-ipfs-provider/simple"
12 ipld "github.com/ipfs/go-ipld-format"
13 "github.com/libp2p/go-libp2p-core/routing"
14 "go.uber.org/fx"
15 +
16 + "github.com/ipfs/go-ipfs/core/node/helpers"
17 + "github.com/ipfs/go-ipfs/repo"
18 )
19
20 const kReprovideFrequency = time.Hour * 12
dagutils/diff.go deleted
-211
@@ -1,211 +0,0 @@
1 -package dagutils
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "path"
7 -
8 - "github.com/ipfs/go-cid"
9 - ipld "github.com/ipfs/go-ipld-format"
10 - dag "github.com/ipfs/go-merkledag"
11 - coreiface "github.com/ipfs/interface-go-ipfs-core"
12 -)
13 -
14 -// These constants define the changes that can be applied to a DAG.
15 -const (
16 - Add = iota
17 - Remove
18 - Mod
19 -)
20 -
21 -// Change represents a change to a DAG and contains a reference to the old and
22 -// new CIDs.
23 -type Change struct {
24 - Type coreiface.ChangeType
25 - Path string
26 - Before cid.Cid
27 - After cid.Cid
28 -}
29 -
30 -// String prints a human-friendly line about a change.
31 -func (c *Change) String() string {
32 - switch c.Type {
33 - case Add:
34 - return fmt.Sprintf("Added %s at %s", c.After.String(), c.Path)
35 - case Remove:
36 - return fmt.Sprintf("Removed %s from %s", c.Before.String(), c.Path)
37 - case Mod:
38 - return fmt.Sprintf("Changed %s to %s at %s", c.Before.String(), c.After.String(), c.Path)
39 - default:
40 - panic("nope")
41 - }
42 -}
43 -
44 -// ApplyChange applies the requested changes to the given node in the given dag.
45 -func ApplyChange(ctx context.Context, ds ipld.DAGService, nd *dag.ProtoNode, cs []*Change) (*dag.ProtoNode, error) {
46 - e := NewDagEditor(nd, ds)
47 - for _, c := range cs {
48 - switch c.Type {
49 - case Add:
50 - child, err := ds.Get(ctx, c.After)
51 - if err != nil {
52 - return nil, err
53 - }
54 -
55 - childpb, ok := child.(*dag.ProtoNode)
56 - if !ok {
57 - return nil, dag.ErrNotProtobuf
58 - }
59 -
60 - err = e.InsertNodeAtPath(ctx, c.Path, childpb, nil)
61 - if err != nil {
62 - return nil, err
63 - }
64 -
65 - case Remove:
66 - err := e.RmLink(ctx, c.Path)
67 - if err != nil {
68 - return nil, err
69 - }
70 -
71 - case Mod:
72 - err := e.RmLink(ctx, c.Path)
73 - if err != nil {
74 - return nil, err
75 - }
76 - child, err := ds.Get(ctx, c.After)
77 - if err != nil {
78 - return nil, err
79 - }
80 -
81 - childpb, ok := child.(*dag.ProtoNode)
82 - if !ok {
83 - return nil, dag.ErrNotProtobuf
84 - }
85 -
86 - err = e.InsertNodeAtPath(ctx, c.Path, childpb, nil)
87 - if err != nil {
88 - return nil, err
89 - }
90 - }
91 - }
92 -
93 - return e.Finalize(ctx, ds)
94 -}
95 -
96 -// Diff returns a set of changes that transform node 'a' into node 'b'.
97 -// It only traverses links in the following cases:
98 -// 1. two node's links number are greater than 0.
99 -// 2. both of two nodes are ProtoNode.
100 -// Otherwise, it compares the cid and emits a Mod change object.
101 -func Diff(ctx context.Context, ds ipld.DAGService, a, b ipld.Node) ([]*Change, error) {
102 - // Base case where both nodes are leaves, just compare
103 - // their CIDs.
104 - if len(a.Links()) == 0 && len(b.Links()) == 0 {
105 - return getChange(a, b)
106 - }
107 -
108 - var out []*Change
109 - cleanA, okA := a.Copy().(*dag.ProtoNode)
110 - cleanB, okB := b.Copy().(*dag.ProtoNode)
111 - if !okA || !okB {
112 - return getChange(a, b)
113 - }
114 -
115 - // strip out unchanged stuff
116 - for _, lnk := range a.Links() {
117 - l, _, err := b.ResolveLink([]string{lnk.Name})
118 - if err == nil {
119 - if l.Cid.Equals(lnk.Cid) {
120 - // no change... ignore it
121 - } else {
122 - anode, err := lnk.GetNode(ctx, ds)
123 - if err != nil {
124 - return nil, err
125 - }
126 -
127 - bnode, err := l.GetNode(ctx, ds)
128 - if err != nil {
129 - return nil, err
130 - }
131 -
132 - sub, err := Diff(ctx, ds, anode, bnode)
133 - if err != nil {
134 - return nil, err
135 - }
136 -
137 - for _, subc := range sub {
138 - subc.Path = path.Join(lnk.Name, subc.Path)
139 - out = append(out, subc)
140 - }
141 - }
142 - _ = cleanA.RemoveNodeLink(l.Name)
143 - _ = cleanB.RemoveNodeLink(l.Name)
144 - }
145 - }
146 -
147 - for _, lnk := range cleanA.Links() {
148 - out = append(out, &Change{
149 - Type: Remove,
150 - Path: lnk.Name,
151 - Before: lnk.Cid,
152 - })
153 - }
154 - for _, lnk := range cleanB.Links() {
155 - out = append(out, &Change{
156 - Type: Add,
157 - Path: lnk.Name,
158 - After: lnk.Cid,
159 - })
160 - }
161 -
162 - return out, nil
163 -}
164 -
165 -// Conflict represents two incompatible changes and is returned by MergeDiffs().
166 -type Conflict struct {
167 - A *Change
168 - B *Change
169 -}
170 -
171 -// MergeDiffs takes two slice of changes and adds them to a single slice.
172 -// When a Change from b happens to the same path of an existing change in a,
173 -// a conflict is created and b is not added to the merged slice.
174 -// A slice of Conflicts is returned and contains pointers to the
175 -// Changes involved (which share the same path).
176 -func MergeDiffs(a, b []*Change) ([]*Change, []Conflict) {
177 - var out []*Change
178 - var conflicts []Conflict
179 - paths := make(map[string]*Change)
180 - for _, c := range a {
181 - paths[c.Path] = c
182 - }
183 -
184 - for _, c := range b {
185 - if ca, ok := paths[c.Path]; ok {
186 - conflicts = append(conflicts, Conflict{
187 - A: ca,
188 - B: c,
189 - })
190 - } else {
191 - out = append(out, c)
192 - }
193 - }
194 - for _, c := range paths {
195 - out = append(out, c)
196 - }
197 - return out, conflicts
198 -}
199 -
200 -func getChange(a, b ipld.Node) ([]*Change, error) {
201 - if a.Cid().Equals(b.Cid()) {
202 - return []*Change{}, nil
203 - }
204 - return []*Change{
205 - {
206 - Type: Mod,
207 - Before: a.Cid(),
208 - After: b.Cid(),
209 - },
210 - }, nil
211 -}
dagutils/diffenum.go deleted
-99
@@ -1,99 +0,0 @@
1 -package dagutils
2 -
3 -import (
4 - "context"
5 - "fmt"
6 -
7 - mdag "github.com/ipfs/go-merkledag"
8 -
9 - cid "github.com/ipfs/go-cid"
10 - ipld "github.com/ipfs/go-ipld-format"
11 -)
12 -
13 -// DiffEnumerate fetches every object in the graph pointed to by 'to' that is
14 -// not in 'from'. This can be used to more efficiently fetch a graph if you can
15 -// guarantee you already have the entirety of 'from'
16 -func DiffEnumerate(ctx context.Context, dserv ipld.NodeGetter, from, to cid.Cid) error {
17 - fnd, err := dserv.Get(ctx, from)
18 - if err != nil {
19 - return fmt.Errorf("get %s: %s", from, err)
20 - }
21 -
22 - tnd, err := dserv.Get(ctx, to)
23 - if err != nil {
24 - return fmt.Errorf("get %s: %s", to, err)
25 - }
26 -
27 - diff := getLinkDiff(fnd, tnd)
28 -
29 - sset := cid.NewSet()
30 - for _, c := range diff {
31 - // Since we're already assuming we have everything in the 'from' graph,
32 - // add all those cids to our 'already seen' set to avoid potentially
33 - // enumerating them later
34 - if c.bef.Defined() {
35 - sset.Add(c.bef)
36 - }
37 - }
38 - for _, c := range diff {
39 - if !c.bef.Defined() {
40 - if sset.Has(c.aft) {
41 - continue
42 - }
43 - err := mdag.Walk(ctx, mdag.GetLinksDirect(dserv), c.aft, sset.Visit, mdag.Concurrent())
44 - if err != nil {
45 - return err
46 - }
47 - } else {
48 - err := DiffEnumerate(ctx, dserv, c.bef, c.aft)
49 - if err != nil {
50 - return err
51 - }
52 - }
53 - }
54 -
55 - return nil
56 -}
57 -
58 -// if both bef and aft are not nil, then that signifies bef was replaces with aft.
59 -// if bef is nil and aft is not, that means aft was newly added
60 -// if aft is nil and bef is not, that means bef was deleted
61 -type diffpair struct {
62 - bef, aft cid.Cid
63 -}
64 -
65 -// getLinkDiff returns a changeset between nodes 'a' and 'b'. Currently does
66 -// not log deletions as our usecase doesnt call for this.
67 -func getLinkDiff(a, b ipld.Node) []diffpair {
68 - ina := make(map[string]*ipld.Link)
69 - inb := make(map[string]*ipld.Link)
70 - var aonly []cid.Cid
71 - for _, l := range b.Links() {
72 - inb[l.Cid.KeyString()] = l
73 - }
74 - for _, l := range a.Links() {
75 - var key = l.Cid.KeyString()
76 - ina[key] = l
77 - if inb[key] == nil {
78 - aonly = append(aonly, l.Cid)
79 - }
80 - }
81 -
82 - var out []diffpair
83 - var aindex int
84 -
85 - for _, l := range b.Links() {
86 - if ina[l.Cid.KeyString()] != nil {
87 - continue
88 - }
89 -
90 - if aindex < len(aonly) {
91 - out = append(out, diffpair{bef: aonly[aindex], aft: l.Cid})
92 - aindex++
93 - } else {
94 - out = append(out, diffpair{aft: l.Cid})
95 - continue
96 - }
97 - }
98 - return out
99 -}
dagutils/diffenum_test.go deleted
-249
@@ -1,249 +0,0 @@
1 -package dagutils
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "testing"
7 -
8 - dag "github.com/ipfs/go-merkledag"
9 - mdtest "github.com/ipfs/go-merkledag/test"
10 -
11 - cid "github.com/ipfs/go-cid"
12 - ipld "github.com/ipfs/go-ipld-format"
13 -)
14 -
15 -func buildNode(name string, desc map[string]ndesc, out map[string]ipld.Node) ipld.Node {
16 - this := desc[name]
17 - nd := new(dag.ProtoNode)
18 - nd.SetData([]byte(name))
19 - for k, v := range this {
20 - child, ok := out[v]
21 - if !ok {
22 - child = buildNode(v, desc, out)
23 - out[v] = child
24 - }
25 -
26 - if err := nd.AddNodeLink(k, child); err != nil {
27 - panic(err)
28 - }
29 - }
30 -
31 - return nd
32 -}
33 -
34 -type ndesc map[string]string
35 -
36 -func mkGraph(desc map[string]ndesc) map[string]ipld.Node {
37 - out := make(map[string]ipld.Node)
38 - for name := range desc {
39 - if _, ok := out[name]; ok {
40 - continue
41 - }
42 -
43 - out[name] = buildNode(name, desc, out)
44 - }
45 - return out
46 -}
47 -
48 -var tg1 = map[string]ndesc{
49 - "a1": ndesc{
50 - "foo": "b",
51 - },
52 - "b": ndesc{},
53 - "a2": ndesc{
54 - "foo": "b",
55 - "bar": "c",
56 - },
57 - "c": ndesc{},
58 -}
59 -
60 -var tg2 = map[string]ndesc{
61 - "a1": ndesc{
62 - "foo": "b",
63 - },
64 - "b": ndesc{},
65 - "a2": ndesc{
66 - "foo": "b",
67 - "bar": "c",
68 - },
69 - "c": ndesc{"baz": "d"},
70 - "d": ndesc{},
71 -}
72 -
73 -var tg3 = map[string]ndesc{
74 - "a1": ndesc{
75 - "foo": "b",
76 - "bar": "c",
77 - },
78 - "b": ndesc{},
79 - "a2": ndesc{
80 - "foo": "b",
81 - "bar": "d",
82 - },
83 - "c": ndesc{},
84 - "d": ndesc{},
85 -}
86 -
87 -var tg4 = map[string]ndesc{
88 - "a1": ndesc{
89 - "key1": "b",
90 - "key2": "c",
91 - },
92 - "a2": ndesc{
93 - "key1": "b",
94 - "key2": "d",
95 - },
96 -}
97 -
98 -var tg5 = map[string]ndesc{
99 - "a1": ndesc{
100 - "key1": "a",
101 - "key2": "b",
102 - },
103 - "a2": ndesc{
104 - "key1": "c",
105 - "key2": "d",
106 - },
107 -}
108 -
109 -func TestNameMatching(t *testing.T) {
110 - nds := mkGraph(tg4)
111 -
112 - diff := getLinkDiff(nds["a1"], nds["a2"])
113 - if len(diff) != 1 {
114 - t.Fatal(fmt.Errorf("node diff didn't match by name"))
115 - }
116 -}
117 -
118 -func TestNameMatching2(t *testing.T) {
119 - nds := mkGraph(tg5)
120 -
121 - diff := getLinkDiff(nds["a1"], nds["a2"])
122 - if len(diff) != 2 {
123 - t.Fatal(fmt.Errorf("incorrect number of link diff elements"))
124 - }
125 - if !(diff[0].bef.Equals(nds["a1"].Links()[0].Cid) && diff[0].aft.Equals(nds["a2"].Links()[0].Cid)) {
126 - t.Fatal(fmt.Errorf("node diff didn't match by name"))
127 - }
128 -}
129 -
130 -func TestDiffEnumBasic(t *testing.T) {
131 - ctx, cancel := context.WithCancel(context.Background())
132 - defer cancel()
133 - nds := mkGraph(tg1)
134 -
135 - ds := mdtest.Mock()
136 - lgds := &getLogger{ds: ds}
137 -
138 - for _, nd := range nds {
139 - err := ds.Add(ctx, nd)
140 - if err != nil {
141 - t.Fatal(err)
142 - }
143 - }
144 -
145 - err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid())
146 - if err != nil {
147 - t.Fatal(err)
148 - }
149 -
150 - err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid()})
151 - if err != nil {
152 - t.Fatal(err)
153 - }
154 -}
155 -
156 -type getLogger struct {
157 - ds ipld.NodeGetter
158 - log []cid.Cid
159 -}
160 -
161 -func (gl *getLogger) Get(ctx context.Context, c cid.Cid) (ipld.Node, error) {
162 - nd, err := gl.ds.Get(ctx, c)
163 - if err != nil {
164 - return nil, err
165 - }
166 - gl.log = append(gl.log, c)
167 - return nd, nil
168 -}
169 -
170 -func (gl *getLogger) GetMany(ctx context.Context, cids []cid.Cid) <-chan *ipld.NodeOption {
171 - outCh := make(chan *ipld.NodeOption, len(cids))
172 - nds := gl.ds.GetMany(ctx, cids)
173 - for no := range nds {
174 - if no.Err == nil {
175 - gl.log = append(gl.log, no.Node.Cid())
176 - }
177 - select {
178 - case outCh <- no:
179 - default:
180 - panic("too many responses")
181 - }
182 - }
183 - return nds
184 -}
185 -
186 -func assertCidList(a, b []cid.Cid) error {
187 - if len(a) != len(b) {
188 - return fmt.Errorf("got different number of cids than expected")
189 - }
190 - for i, c := range a {
191 - if !c.Equals(b[i]) {
192 - return fmt.Errorf("expected %s, got %s", c, b[i])
193 - }
194 - }
195 - return nil
196 -}
197 -
198 -func TestDiffEnumFail(t *testing.T) {
199 - ctx, cancel := context.WithCancel(context.Background())
200 - defer cancel()
201 - nds := mkGraph(tg2)
202 -
203 - ds := mdtest.Mock()
204 - lgds := &getLogger{ds: ds}
205 -
206 - for _, s := range []string{"a1", "a2", "b", "c"} {
207 - err := ds.Add(ctx, nds[s])
208 - if err != nil {
209 - t.Fatal(err)
210 - }
211 - }
212 -
213 - err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid())
214 - if err != ipld.ErrNotFound {
215 - t.Fatal("expected err not found")
216 - }
217 -
218 - err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid()})
219 - if err != nil {
220 - t.Fatal(err)
221 - }
222 -
223 -}
224 -
225 -func TestDiffEnumRecurse(t *testing.T) {
226 - ctx, cancel := context.WithCancel(context.Background())
227 - defer cancel()
228 - nds := mkGraph(tg3)
229 -
230 - ds := mdtest.Mock()
231 - lgds := &getLogger{ds: ds}
232 -
233 - for _, s := range []string{"a1", "a2", "b", "c", "d"} {
234 - err := ds.Add(ctx, nds[s])
235 - if err != nil {
236 - t.Fatal(err)
237 - }
238 - }
239 -
240 - err := DiffEnumerate(ctx, lgds, nds["a1"].Cid(), nds["a2"].Cid())
241 - if err != nil {
242 - t.Fatal(err)
243 - }
244 -
245 - err = assertCidList(lgds.log, []cid.Cid{nds["a1"].Cid(), nds["a2"].Cid(), nds["c"].Cid(), nds["d"].Cid()})
246 - if err != nil {
247 - t.Fatal(err)
248 - }
249 -}
dagutils/utils.go deleted
-234
@@ -1,234 +0,0 @@
1 -package dagutils
2 -
3 -import (
4 - "context"
5 - "errors"
6 -
7 - bserv "github.com/ipfs/go-blockservice"
8 - dag "github.com/ipfs/go-merkledag"
9 - path "github.com/ipfs/go-path"
10 -
11 - ds "github.com/ipfs/go-datastore"
12 - syncds "github.com/ipfs/go-datastore/sync"
13 - bstore "github.com/ipfs/go-ipfs-blockstore"
14 - offline "github.com/ipfs/go-ipfs-exchange-offline"
15 - ipld "github.com/ipfs/go-ipld-format"
16 -)
17 -
18 -// Editor represents a ProtoNode tree editor and provides methods to
19 -// modify it.
20 -type Editor struct {
21 - root *dag.ProtoNode
22 -
23 - // tmp is a temporary in memory (for now) dagstore for all of the
24 - // intermediary nodes to be stored in
25 - tmp ipld.DAGService
26 -
27 - // src is the dagstore with *all* of the data on it, it is used to pull
28 - // nodes from for modification (nil is a valid value)
29 - src ipld.DAGService
30 -}
31 -
32 -// NewMemoryDagService returns a new, thread-safe in-memory DAGService.
33 -func NewMemoryDagService() ipld.DAGService {
34 - // build mem-datastore for editor's intermediary nodes
35 - bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
36 - bsrv := bserv.New(bs, offline.Exchange(bs))
37 - return dag.NewDAGService(bsrv)
38 -}
39 -
40 -// NewDagEditor returns an ProtoNode editor.
41 -//
42 -// * root is the node to be modified
43 -// * source is the dagstore to pull nodes from (optional)
44 -func NewDagEditor(root *dag.ProtoNode, source ipld.DAGService) *Editor {
45 - return &Editor{
46 - root: root,
47 - tmp: NewMemoryDagService(),
48 - src: source,
49 - }
50 -}
51 -
52 -// GetNode returns the a copy of the root node being edited.
53 -func (e *Editor) GetNode() *dag.ProtoNode {
54 - return e.root.Copy().(*dag.ProtoNode)
55 -}
56 -
57 -// GetDagService returns the DAGService used by this editor.
58 -func (e *Editor) GetDagService() ipld.DAGService {
59 - return e.tmp
60 -}
61 -
62 -func addLink(ctx context.Context, ds ipld.DAGService, root *dag.ProtoNode, childname string, childnd ipld.Node) (*dag.ProtoNode, error) {
63 - if childname == "" {
64 - return nil, errors.New("cannot create link with no name")
65 - }
66 -
67 - // ensure that the node we are adding is in the dagservice
68 - err := ds.Add(ctx, childnd)
69 - if err != nil {
70 - return nil, err
71 - }
72 -
73 - _ = ds.Remove(ctx, root.Cid())
74 -
75 - // ensure no link with that name already exists
76 - _ = root.RemoveNodeLink(childname) // ignore error, only option is ErrNotFound
77 -
78 - if err := root.AddNodeLink(childname, childnd); err != nil {
79 - return nil, err
80 - }
81 -
82 - if err := ds.Add(ctx, root); err != nil {
83 - return nil, err
84 - }
85 - return root, nil
86 -}
87 -
88 -// InsertNodeAtPath inserts a new node in the tree and replaces the current root with the new one.
89 -func (e *Editor) InsertNodeAtPath(ctx context.Context, pth string, toinsert ipld.Node, create func() *dag.ProtoNode) error {
90 - splpath := path.SplitList(pth)
91 - nd, err := e.insertNodeAtPath(ctx, e.root, splpath, toinsert, create)
92 - if err != nil {
93 - return err
94 - }
95 - e.root = nd
96 - return nil
97 -}
98 -
99 -func (e *Editor) insertNodeAtPath(ctx context.Context, root *dag.ProtoNode, path []string, toinsert ipld.Node, create func() *dag.ProtoNode) (*dag.ProtoNode, error) {
100 - if len(path) == 1 {
101 - return addLink(ctx, e.tmp, root, path[0], toinsert)
102 - }
103 -
104 - nd, err := root.GetLinkedProtoNode(ctx, e.tmp, path[0])
105 - if err != nil {
106 - // if 'create' is true, we create directories on the way down as needed
107 - if err == dag.ErrLinkNotFound && create != nil {
108 - nd = create()
109 - err = nil // no longer an error case
110 - } else if err == ipld.ErrNotFound {
111 - // try finding it in our source dagstore
112 - nd, err = root.GetLinkedProtoNode(ctx, e.src, path[0])
113 - }
114 -
115 - // if we receive an ErrNotFound, then our second 'GetLinkedNode' call
116 - // also fails, we want to error out
117 - if err != nil {
118 - return nil, err
119 - }
120 - }
121 -
122 - ndprime, err := e.insertNodeAtPath(ctx, nd, path[1:], toinsert, create)
123 - if err != nil {
124 - return nil, err
125 - }
126 -
127 - _ = e.tmp.Remove(ctx, root.Cid())
128 -
129 - _ = root.RemoveNodeLink(path[0])
130 - err = root.AddNodeLink(path[0], ndprime)
131 - if err != nil {
132 - return nil, err
133 - }
134 -
135 - err = e.tmp.Add(ctx, root)
136 - if err != nil {
137 - return nil, err
138 - }
139 -
140 - return root, nil
141 -}
142 -
143 -// RmLink removes the link with the given name and updates the root node of
144 -// the editor.
145 -func (e *Editor) RmLink(ctx context.Context, pth string) error {
146 - splpath := path.SplitList(pth)
147 - nd, err := e.rmLink(ctx, e.root, splpath)
148 - if err != nil {
149 - return err
150 - }
151 - e.root = nd
152 - return nil
153 -}
154 -
155 -func (e *Editor) rmLink(ctx context.Context, root *dag.ProtoNode, path []string) (*dag.ProtoNode, error) {
156 - if len(path) == 1 {
157 - // base case, remove node in question
158 - err := root.RemoveNodeLink(path[0])
159 - if err != nil {
160 - return nil, err
161 - }
162 -
163 - err = e.tmp.Add(ctx, root)
164 - if err != nil {
165 - return nil, err
166 - }
167 -
168 - return root, nil
169 - }
170 -
171 - // search for node in both tmp dagstore and source dagstore
172 - nd, err := root.GetLinkedProtoNode(ctx, e.tmp, path[0])
173 - if err == ipld.ErrNotFound {
174 - nd, err = root.GetLinkedProtoNode(ctx, e.src, path[0])
175 - }
176 -
177 - if err != nil {
178 - return nil, err
179 - }
180 -
181 - nnode, err := e.rmLink(ctx, nd, path[1:])
182 - if err != nil {
183 - return nil, err
184 - }
185 -
186 - _ = e.tmp.Remove(ctx, root.Cid())
187 -
188 - _ = root.RemoveNodeLink(path[0])
189 - err = root.AddNodeLink(path[0], nnode)
190 - if err != nil {
191 - return nil, err
192 - }
193 -
194 - err = e.tmp.Add(ctx, root)
195 - if err != nil {
196 - return nil, err
197 - }
198 -
199 - return root, nil
200 -}
201 -
202 -// Finalize writes the new DAG to the given DAGService and returns the modified
203 -// root node.
204 -func (e *Editor) Finalize(ctx context.Context, ds ipld.DAGService) (*dag.ProtoNode, error) {
205 - nd := e.GetNode()
206 - err := copyDag(ctx, nd, e.tmp, ds)
207 - return nd, err
208 -}
209 -
210 -func copyDag(ctx context.Context, nd ipld.Node, from, to ipld.DAGService) error {
211 - // TODO(#4609): make this batch.
212 - err := to.Add(ctx, nd)
213 - if err != nil {
214 - return err
215 - }
216 -
217 - for _, lnk := range nd.Links() {
218 - child, err := lnk.GetNode(ctx, from)
219 - if err != nil {
220 - if err == ipld.ErrNotFound {
221 - // not found means we didnt modify it, and it should
222 - // already be in the target datastore
223 - continue
224 - }
225 - return err
226 - }
227 -
228 - err = copyDag(ctx, child, from, to)
229 - if err != nil {
230 - return err
231 - }
232 - }
233 - return nil
234 -}
dagutils/utils_test.go deleted
-114
@@ -1,114 +0,0 @@
1 -package dagutils
2 -
3 -import (
4 - "context"
5 - "testing"
6 -
7 - dag "github.com/ipfs/go-merkledag"
8 - mdtest "github.com/ipfs/go-merkledag/test"
9 - path "github.com/ipfs/go-path"
10 -
11 - cid "github.com/ipfs/go-cid"
12 - ipld "github.com/ipfs/go-ipld-format"
13 -)
14 -
15 -func TestAddLink(t *testing.T) {
16 - ctx, context := context.WithCancel(context.Background())
17 - defer context()
18 -
19 - ds := mdtest.Mock()
20 - fishnode := dag.NodeWithData([]byte("fishcakes!"))
21 -
22 - err := ds.Add(ctx, fishnode)
23 - if err != nil {
24 - t.Fatal(err)
25 - }
26 -
27 - nd := new(dag.ProtoNode)
28 - nnode, err := addLink(ctx, ds, nd, "fish", fishnode)
29 - if err != nil {
30 - t.Fatal(err)
31 - }
32 -
33 - fnprime, err := nnode.GetLinkedNode(ctx, ds, "fish")
34 - if err != nil {
35 - t.Fatal(err)
36 - }
37 -
38 - fnpkey := fnprime.Cid()
39 - if !fnpkey.Equals(fishnode.Cid()) {
40 - t.Fatal("wrong child node found!")
41 - }
42 -}
43 -
44 -func assertNodeAtPath(t *testing.T, ds ipld.DAGService, root *dag.ProtoNode, pth string, exp cid.Cid) {
45 - parts := path.SplitList(pth)
46 - cur := root
47 - for _, e := range parts {
48 - nxt, err := cur.GetLinkedProtoNode(context.Background(), ds, e)
49 - if err != nil {
50 - t.Fatal(err)
51 - }
52 -
53 - cur = nxt
54 - }
55 -
56 - curc := cur.Cid()
57 - if !curc.Equals(exp) {
58 - t.Fatal("node not as expected at end of path")
59 - }
60 -}
61 -
62 -func TestInsertNode(t *testing.T) {
63 - root := new(dag.ProtoNode)
64 - e := NewDagEditor(root, nil)
65 -
66 - testInsert(t, e, "a", "anodefortesting", false, "")
67 - testInsert(t, e, "a/b", "data", false, "")
68 - testInsert(t, e, "a/b/c/d/e", "blah", false, "no link by that name")
69 - testInsert(t, e, "a/b/c/d/e", "foo", true, "")
70 - testInsert(t, e, "a/b/c/d/f", "baz", true, "")
71 - testInsert(t, e, "a/b/c/d/f", "bar", true, "")
72 -
73 - testInsert(t, e, "", "bar", true, "cannot create link with no name")
74 - testInsert(t, e, "////", "slashes", true, "cannot create link with no name")
75 -
76 - c := e.GetNode().Cid()
77 -
78 - if c.String() != "QmZ8yeT9uD6ouJPNAYt62XffYuXBT6b4mP4obRSE9cJrSt" {
79 - t.Fatal("output was different than expected: ", c)
80 - }
81 -}
82 -
83 -func testInsert(t *testing.T, e *Editor, path, data string, create bool, experr string) {
84 - child := dag.NodeWithData([]byte(data))
85 - err := e.tmp.Add(context.Background(), child)
86 - if err != nil {
87 - t.Fatal(err)
88 - }
89 -
90 - var c func() *dag.ProtoNode
91 - if create {
92 - c = func() *dag.ProtoNode {
93 - return &dag.ProtoNode{}
94 - }
95 - }
96 -
97 - err = e.InsertNodeAtPath(context.Background(), path, child, c)
98 - if experr != "" {
99 - var got string
100 - if err != nil {
101 - got = err.Error()
102 - }
103 - if got != experr {
104 - t.Fatalf("expected '%s' but got '%s'", experr, got)
105 - }
106 - return
107 - }
108 -
109 - if err != nil {
110 - t.Fatal(err, path, data, create, experr)
111 - }
112 -
113 - assertNodeAtPath(t, e.tmp, e.root, path, child.Cid())
114 -}
gc/gc.go renamed
+2 -3
@@ -8,15 +8,14 @@ import (
8 "strings"
9
10 bserv "github.com/ipfs/go-blockservice"
11 - pin "github.com/ipfs/go-ipfs/pin"
12 - dag "github.com/ipfs/go-merkledag"
13 -
11 cid "github.com/ipfs/go-cid"
12 dstore "github.com/ipfs/go-datastore"
13 bstore "github.com/ipfs/go-ipfs-blockstore"
14 offline "github.com/ipfs/go-ipfs-exchange-offline"
15 + pin "github.com/ipfs/go-ipfs-pinner"
16 ipld "github.com/ipfs/go-ipld-format"
17 logging "github.com/ipfs/go-log"
18 + dag "github.com/ipfs/go-merkledag"
19 "github.com/ipfs/go-verifcid"
20 )
21
go.mod
+3 -2
@@ -35,6 +35,7 @@ require (
35 github.com/ipfs/go-ipfs-exchange-interface v0.0.1
36 github.com/ipfs/go-ipfs-exchange-offline v0.0.1
37 github.com/ipfs/go-ipfs-files v0.0.4
38 + github.com/ipfs/go-ipfs-pinner v0.0.2
39 github.com/ipfs/go-ipfs-posinfo v0.0.1
40 github.com/ipfs/go-ipfs-provider v0.3.0
41 github.com/ipfs/go-ipfs-routing v0.1.0
@@ -44,7 +45,7 @@ require (
45 github.com/ipfs/go-ipld-git v0.0.2
46 github.com/ipfs/go-ipns v0.0.1
47 github.com/ipfs/go-log v0.0.1
47 - github.com/ipfs/go-merkledag v0.2.3
48 + github.com/ipfs/go-merkledag v0.3.0
49 github.com/ipfs/go-metrics-interface v0.0.1
50 github.com/ipfs/go-metrics-prometheus v0.0.2
51 github.com/ipfs/go-mfs v0.1.1
@@ -107,4 +108,4 @@ require (
108 gopkg.in/cheggaaa/pb.v1 v1.0.28
109 )
110
110 -go 1.13
111 +go 1.12
go.sum
+14
@@ -8,6 +8,10 @@ github.com/AndreasBriese/bbloom v0.0.0-20190823232136-616930265c33/go.mod h1:bOv
8 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
9 github.com/Kubuxu/go-os-helper v0.0.1 h1:EJiD2VUQyh5A9hWJLmc6iWg6yIcJ7jpBcwC8GMGXfDk=
10 github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y=
11 +github.com/MichaelMure/go-ipfs-pinner v0.0.0-20191120152055-f93a308647da h1:VujEWo1PUA+EIMvYU26fNvqykLMBuyXkILKbue86bzg=
12 +github.com/MichaelMure/go-ipfs-pinner v0.0.0-20191120152055-f93a308647da/go.mod h1:0G0RKJr9eZwNvcwdLNVymjCcFhWtX9aXVGOxCZglwMg=
13 +github.com/MichaelMure/go-merkledag v0.2.1-0.20191119160700-c20b9a52f504 h1:2qPGrw2YNfWNaOTPHY/WiSQZUKgiGQWh34my2zSppzo=
14 +github.com/MichaelMure/go-merkledag v0.2.1-0.20191119160700-c20b9a52f504/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
15 github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo=
16 github.com/Stebalien/go-bitfield v0.0.1 h1:X3kbSSPUaJK60wV2hjOPZwmpljr6VGCqdq4cBLhbQBo=
17 github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s=
@@ -209,6 +213,8 @@ github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjN
213 github.com/ipfs/go-ipfs-files v0.0.4 h1:WzRCivcybUQch/Qh6v8LBRhKtRsjnwyiuOV09mK7mrE=
214 github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
215 github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA=
216 +github.com/ipfs/go-ipfs-pinner v0.0.2 h1:KRXt2V0TzoTd3mO1aONSw8C9wnZtl7RLpPruN/XDnlQ=
217 +github.com/ipfs/go-ipfs-pinner v0.0.2/go.mod h1:KZGyGAR+yLthGEkG9tuA2zweB7O6auXaJNjX6IbEbOs=
218 github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs=
219 github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A=
220 github.com/ipfs/go-ipfs-pq v0.0.1 h1:zgUotX8dcAB/w/HidJh1zzc1yFq6Vm8J7T2F4itj/RU=
@@ -238,6 +244,9 @@ github.com/ipfs/go-merkledag v0.0.6/go.mod h1:QYPdnlvkOg7GnQRofu9XZimC5ZW5Wi3bKy
244 github.com/ipfs/go-merkledag v0.1.0/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
245 github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE=
246 github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
247 +github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk=
248 +github.com/ipfs/go-merkledag v0.3.0 h1:1bXv/ZRPZLVdij/a33CkXMVdxUdred9sz4xyph+0ls0=
249 +github.com/ipfs/go-merkledag v0.3.0/go.mod h1:4pymaZLhSLNVuiCITYrpViD6vmfZ/Ws4n/L9tfNv3S4=
250 github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg=
251 github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY=
252 github.com/ipfs/go-metrics-prometheus v0.0.2 h1:9i2iljLg12S78OhC6UAiXi176xvQGiZaGVF1CUVdE+s=
@@ -259,9 +268,14 @@ github.com/ipfs/go-unixfs v0.2.1 h1:g51t9ODICFZ3F51FPivm8dE7NzYcdAQNUL9wGP5AYa0=
268 github.com/ipfs/go-unixfs v0.2.1/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k=
269 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
270 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
271 +github.com/ipfs/interface-go-ipfs-core v0.2.3 h1:E6uQ+1fJjkxJWlL9lAE72a5FWeyeeNL3GitLy8+jq3Y=
272 +github.com/ipfs/interface-go-ipfs-core v0.2.3 h1:E6uQ+1fJjkxJWlL9lAE72a5FWeyeeNL3GitLy8+jq3Y=
273 +github.com/ipfs/interface-go-ipfs-core v0.2.3/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s=
274 +github.com/ipfs/interface-go-ipfs-core v0.2.3/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s=
275 github.com/ipfs/interface-go-ipfs-core v0.2.5 h1:/rspOe8RbIxwtssEXHB+X9JXhOBDCQt8x50d2kFPXL8=
276 github.com/ipfs/interface-go-ipfs-core v0.2.5/go.mod h1:Tihp8zxGpUeE3Tokr94L6zWZZdkRQvG5TL6i9MuNE+s=
277 github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
278 +github.com/jackpal/gateway v1.0.4/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
279 github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc=
280 github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA=
281 github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA=
namesys/publisher.go
+1 -2
@@ -6,11 +6,10 @@ import (
6 "sync"
7 "time"
8
9 - pin "github.com/ipfs/go-ipfs/pin"
10 -
9 proto "github.com/gogo/protobuf/proto"
10 ds "github.com/ipfs/go-datastore"
11 dsquery "github.com/ipfs/go-datastore/query"
12 + pin "github.com/ipfs/go-ipfs-pinner"
13 ipns "github.com/ipfs/go-ipns"
14 pb "github.com/ipfs/go-ipns/pb"
15 path "github.com/ipfs/go-path"
pin/internal/pb/Rules.mk deleted
-8
@@ -1,8 +0,0 @@
1 -include mk/header.mk
2 -
3 -PB_$(d) = $(wildcard $(d)/*.proto)
4 -TGTS_$(d) = $(PB_$(d):.proto=.pb.go)
5 -
6 -#DEPS_GO += $(TGTS_$(d))
7 -
8 -include mk/footer.mk
pin/internal/pb/doc.go deleted
-3
@@ -1,3 +0,0 @@
1 -package pb
2 -
3 -//go:generate protoc --gogo_out=. header.proto
pin/internal/pb/header.pb.go deleted
-381
@@ -1,381 +0,0 @@
1 -// Code generated by protoc-gen-gogo. DO NOT EDIT.
2 -// source: pin/internal/pb/header.proto
3 -
4 -package pb
5 -
6 -import (
7 - encoding_binary "encoding/binary"
8 - fmt "fmt"
9 - proto "github.com/gogo/protobuf/proto"
10 - io "io"
11 - math "math"
12 -)
13 -
14 -// Reference imports to suppress errors if they are not otherwise used.
15 -var _ = proto.Marshal
16 -var _ = fmt.Errorf
17 -var _ = math.Inf
18 -
19 -// This is a compile-time assertion to ensure that this generated file
20 -// is compatible with the proto package it is being compiled against.
21 -// A compilation error at this line likely means your copy of the
22 -// proto package needs to be updated.
23 -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
24 -
25 -type Set struct {
26 - // 1 for now, library will refuse to handle entries with an unrecognized version.
27 - Version uint32 `protobuf:"varint,1,opt,name=version" json:"version"`
28 - // how many of the links are subtrees
29 - Fanout uint32 `protobuf:"varint,2,opt,name=fanout" json:"fanout"`
30 - // hash seed for subtree selection, a random number
31 - Seed uint32 `protobuf:"fixed32,3,opt,name=seed" json:"seed"`
32 -}
33 -
34 -func (m *Set) Reset() { *m = Set{} }
35 -func (m *Set) String() string { return proto.CompactTextString(m) }
36 -func (*Set) ProtoMessage() {}
37 -func (*Set) Descriptor() ([]byte, []int) {
38 - return fileDescriptor_cda303a5a3ed87e7, []int{0}
39 -}
40 -func (m *Set) XXX_Unmarshal(b []byte) error {
41 - return m.Unmarshal(b)
42 -}
43 -func (m *Set) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
44 - if deterministic {
45 - return xxx_messageInfo_Set.Marshal(b, m, deterministic)
46 - } else {
47 - b = b[:cap(b)]
48 - n, err := m.MarshalTo(b)
49 - if err != nil {
50 - return nil, err
51 - }
52 - return b[:n], nil
53 - }
54 -}
55 -func (m *Set) XXX_Merge(src proto.Message) {
56 - xxx_messageInfo_Set.Merge(m, src)
57 -}
58 -func (m *Set) XXX_Size() int {
59 - return m.Size()
60 -}
61 -func (m *Set) XXX_DiscardUnknown() {
62 - xxx_messageInfo_Set.DiscardUnknown(m)
63 -}
64 -
65 -var xxx_messageInfo_Set proto.InternalMessageInfo
66 -
67 -func (m *Set) GetVersion() uint32 {
68 - if m != nil {
69 - return m.Version
70 - }
71 - return 0
72 -}
73 -
74 -func (m *Set) GetFanout() uint32 {
75 - if m != nil {
76 - return m.Fanout
77 - }
78 - return 0
79 -}
80 -
81 -func (m *Set) GetSeed() uint32 {
82 - if m != nil {
83 - return m.Seed
84 - }
85 - return 0
86 -}
87 -
88 -func init() {
89 - proto.RegisterType((*Set)(nil), "ipfs.pin.Set")
90 -}
91 -
92 -func init() { proto.RegisterFile("pin/internal/pb/header.proto", fileDescriptor_cda303a5a3ed87e7) }
93 -
94 -var fileDescriptor_cda303a5a3ed87e7 = []byte{
95 - // 162 bytes of a gzipped FileDescriptorProto
96 - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x29, 0xc8, 0xcc, 0xd3,
97 - 0xcf, 0xcc, 0x2b, 0x49, 0x2d, 0xca, 0x4b, 0xcc, 0xd1, 0x2f, 0x48, 0xd2, 0xcf, 0x48, 0x4d, 0x4c,
98 - 0x49, 0x2d, 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0xc8, 0x2c, 0x48, 0x2b, 0xd6, 0x2b,
99 - 0xc8, 0xcc, 0x53, 0x8a, 0xe5, 0x62, 0x0e, 0x4e, 0x2d, 0x11, 0x92, 0xe3, 0x62, 0x2f, 0x4b, 0x2d,
100 - 0x2a, 0xce, 0xcc, 0xcf, 0x93, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x75, 0x62, 0x39, 0x71, 0x4f, 0x9e,
101 - 0x21, 0x08, 0x26, 0x28, 0x24, 0xc3, 0xc5, 0x96, 0x96, 0x98, 0x97, 0x5f, 0x5a, 0x22, 0xc1, 0x84,
102 - 0x24, 0x0d, 0x15, 0x13, 0x92, 0xe0, 0x62, 0x29, 0x4e, 0x4d, 0x4d, 0x91, 0x60, 0x56, 0x60, 0xd4,
103 - 0x60, 0x87, 0xca, 0x81, 0x45, 0x9c, 0x64, 0x4e, 0x3c, 0x92, 0x63, 0xbc, 0xf0, 0x48, 0x8e, 0xf1,
104 - 0xc1, 0x23, 0x39, 0xc6, 0x09, 0x8f, 0xe5, 0x18, 0x2e, 0x3c, 0x96, 0x63, 0xb8, 0xf1, 0x58, 0x8e,
105 - 0x21, 0x8a, 0xa9, 0x20, 0x09, 0x10, 0x00, 0x00, 0xff, 0xff, 0x20, 0x85, 0x2f, 0x24, 0xa5, 0x00,
106 - 0x00, 0x00,
107 -}
108 -
109 -func (m *Set) Marshal() (dAtA []byte, err error) {
110 - size := m.Size()
111 - dAtA = make([]byte, size)
112 - n, err := m.MarshalTo(dAtA)
113 - if err != nil {
114 - return nil, err
115 - }
116 - return dAtA[:n], nil
117 -}
118 -
119 -func (m *Set) MarshalTo(dAtA []byte) (int, error) {
120 - var i int
121 - _ = i
122 - var l int
123 - _ = l
124 - dAtA[i] = 0x8
125 - i++
126 - i = encodeVarintHeader(dAtA, i, uint64(m.Version))
127 - dAtA[i] = 0x10
128 - i++
129 - i = encodeVarintHeader(dAtA, i, uint64(m.Fanout))
130 - dAtA[i] = 0x1d
131 - i++
132 - encoding_binary.LittleEndian.PutUint32(dAtA[i:], uint32(m.Seed))
133 - i += 4
134 - return i, nil
135 -}
136 -
137 -func encodeVarintHeader(dAtA []byte, offset int, v uint64) int {
138 - for v >= 1<<7 {
139 - dAtA[offset] = uint8(v&0x7f | 0x80)
140 - v >>= 7
141 - offset++
142 - }
143 - dAtA[offset] = uint8(v)
144 - return offset + 1
145 -}
146 -func (m *Set) Size() (n int) {
147 - if m == nil {
148 - return 0
149 - }
150 - var l int
151 - _ = l
152 - n += 1 + sovHeader(uint64(m.Version))
153 - n += 1 + sovHeader(uint64(m.Fanout))
154 - n += 5
155 - return n
156 -}
157 -
158 -func sovHeader(x uint64) (n int) {
159 - for {
160 - n++
161 - x >>= 7
162 - if x == 0 {
163 - break
164 - }
165 - }
166 - return n
167 -}
168 -func sozHeader(x uint64) (n int) {
169 - return sovHeader(uint64((x << 1) ^ uint64((int64(x) >> 63))))
170 -}
171 -func (m *Set) Unmarshal(dAtA []byte) error {
172 - l := len(dAtA)
173 - iNdEx := 0
174 - for iNdEx < l {
175 - preIndex := iNdEx
176 - var wire uint64
177 - for shift := uint(0); ; shift += 7 {
178 - if shift >= 64 {
179 - return ErrIntOverflowHeader
180 - }
181 - if iNdEx >= l {
182 - return io.ErrUnexpectedEOF
183 - }
184 - b := dAtA[iNdEx]
185 - iNdEx++
186 - wire |= uint64(b&0x7F) << shift
187 - if b < 0x80 {
188 - break
189 - }
190 - }
191 - fieldNum := int32(wire >> 3)
192 - wireType := int(wire & 0x7)
193 - if wireType == 4 {
194 - return fmt.Errorf("proto: Set: wiretype end group for non-group")
195 - }
196 - if fieldNum <= 0 {
197 - return fmt.Errorf("proto: Set: illegal tag %d (wire type %d)", fieldNum, wire)
198 - }
199 - switch fieldNum {
200 - case 1:
201 - if wireType != 0 {
202 - return fmt.Errorf("proto: wrong wireType = %d for field Version", wireType)
203 - }
204 - m.Version = 0
205 - for shift := uint(0); ; shift += 7 {
206 - if shift >= 64 {
207 - return ErrIntOverflowHeader
208 - }
209 - if iNdEx >= l {
210 - return io.ErrUnexpectedEOF
211 - }
212 - b := dAtA[iNdEx]
213 - iNdEx++
214 - m.Version |= uint32(b&0x7F) << shift
215 - if b < 0x80 {
216 - break
217 - }
218 - }
219 - case 2:
220 - if wireType != 0 {
221 - return fmt.Errorf("proto: wrong wireType = %d for field Fanout", wireType)
222 - }
223 - m.Fanout = 0
224 - for shift := uint(0); ; shift += 7 {
225 - if shift >= 64 {
226 - return ErrIntOverflowHeader
227 - }
228 - if iNdEx >= l {
229 - return io.ErrUnexpectedEOF
230 - }
231 - b := dAtA[iNdEx]
232 - iNdEx++
233 - m.Fanout |= uint32(b&0x7F) << shift
234 - if b < 0x80 {
235 - break
236 - }
237 - }
238 - case 3:
239 - if wireType != 5 {
240 - return fmt.Errorf("proto: wrong wireType = %d for field Seed", wireType)
241 - }
242 - m.Seed = 0
243 - if (iNdEx + 4) > l {
244 - return io.ErrUnexpectedEOF
245 - }
246 - m.Seed = uint32(encoding_binary.LittleEndian.Uint32(dAtA[iNdEx:]))
247 - iNdEx += 4
248 - default:
249 - iNdEx = preIndex
250 - skippy, err := skipHeader(dAtA[iNdEx:])
251 - if err != nil {
252 - return err
253 - }
254 - if skippy < 0 {
255 - return ErrInvalidLengthHeader
256 - }
257 - if (iNdEx + skippy) < 0 {
258 - return ErrInvalidLengthHeader
259 - }
260 - if (iNdEx + skippy) > l {
261 - return io.ErrUnexpectedEOF
262 - }
263 - iNdEx += skippy
264 - }
265 - }
266 -
267 - if iNdEx > l {
268 - return io.ErrUnexpectedEOF
269 - }
270 - return nil
271 -}
272 -func skipHeader(dAtA []byte) (n int, err error) {
273 - l := len(dAtA)
274 - iNdEx := 0
275 - for iNdEx < l {
276 - var wire uint64
277 - for shift := uint(0); ; shift += 7 {
278 - if shift >= 64 {
279 - return 0, ErrIntOverflowHeader
280 - }
281 - if iNdEx >= l {
282 - return 0, io.ErrUnexpectedEOF
283 - }
284 - b := dAtA[iNdEx]
285 - iNdEx++
286 - wire |= (uint64(b) & 0x7F) << shift
287 - if b < 0x80 {
288 - break
289 - }
290 - }
291 - wireType := int(wire & 0x7)
292 - switch wireType {
293 - case 0:
294 - for shift := uint(0); ; shift += 7 {
295 - if shift >= 64 {
296 - return 0, ErrIntOverflowHeader
297 - }
298 - if iNdEx >= l {
299 - return 0, io.ErrUnexpectedEOF
300 - }
301 - iNdEx++
302 - if dAtA[iNdEx-1] < 0x80 {
303 - break
304 - }
305 - }
306 - return iNdEx, nil
307 - case 1:
308 - iNdEx += 8
309 - return iNdEx, nil
310 - case 2:
311 - var length int
312 - for shift := uint(0); ; shift += 7 {
313 - if shift >= 64 {
314 - return 0, ErrIntOverflowHeader
315 - }
316 - if iNdEx >= l {
317 - return 0, io.ErrUnexpectedEOF
318 - }
319 - b := dAtA[iNdEx]
320 - iNdEx++
321 - length |= (int(b) & 0x7F) << shift
322 - if b < 0x80 {
323 - break
324 - }
325 - }
326 - if length < 0 {
327 - return 0, ErrInvalidLengthHeader
328 - }
329 - iNdEx += length
330 - if iNdEx < 0 {
331 - return 0, ErrInvalidLengthHeader
332 - }
333 - return iNdEx, nil
334 - case 3:
335 - for {
336 - var innerWire uint64
337 - var start int = iNdEx
338 - for shift := uint(0); ; shift += 7 {
339 - if shift >= 64 {
340 - return 0, ErrIntOverflowHeader
341 - }
342 - if iNdEx >= l {
343 - return 0, io.ErrUnexpectedEOF
344 - }
345 - b := dAtA[iNdEx]
346 - iNdEx++
347 - innerWire |= (uint64(b) & 0x7F) << shift
348 - if b < 0x80 {
349 - break
350 - }
351 - }
352 - innerWireType := int(innerWire & 0x7)
353 - if innerWireType == 4 {
354 - break
355 - }
356 - next, err := skipHeader(dAtA[start:])
357 - if err != nil {
358 - return 0, err
359 - }
360 - iNdEx = start + next
361 - if iNdEx < 0 {
362 - return 0, ErrInvalidLengthHeader
363 - }
364 - }
365 - return iNdEx, nil
366 - case 4:
367 - return iNdEx, nil
368 - case 5:
369 - iNdEx += 4
370 - return iNdEx, nil
371 - default:
372 - return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
373 - }
374 - }
375 - panic("unreachable")
376 -}
377 -
378 -var (
379 - ErrInvalidLengthHeader = fmt.Errorf("proto: negative length found during unmarshaling")
380 - ErrIntOverflowHeader = fmt.Errorf("proto: integer overflow")
381 -)
pin/internal/pb/header.proto deleted
-14
@@ -1,14 +0,0 @@
1 -syntax = "proto2";
2 -
3 -package ipfs.pin;
4 -
5 -option go_package = "pb";
6 -
7 -message Set {
8 - // 1 for now, library will refuse to handle entries with an unrecognized version.
9 - optional uint32 version = 1;
10 - // how many of the links are subtrees
11 - optional uint32 fanout = 2;
12 - // hash seed for subtree selection, a random number
13 - optional fixed32 seed = 3;
14 -}
pin/pin.go deleted
-634
@@ -1,634 +0,0 @@
1 -// Package pin implements structures and methods to keep track of
2 -// which objects a user wants to keep stored locally.
3 -package pin
4 -
5 -import (
6 - "context"
7 - "fmt"
8 - "os"
9 - "sync"
10 - "time"
11 -
12 - "github.com/ipfs/go-ipfs/dagutils"
13 - mdag "github.com/ipfs/go-merkledag"
14 -
15 - cid "github.com/ipfs/go-cid"
16 - ds "github.com/ipfs/go-datastore"
17 - ipld "github.com/ipfs/go-ipld-format"
18 - logging "github.com/ipfs/go-log"
19 -)
20 -
21 -var log = logging.Logger("pin")
22 -
23 -var pinDatastoreKey = ds.NewKey("/local/pins")
24 -
25 -var emptyKey cid.Cid
26 -
27 -func init() {
28 - e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n")
29 - if err != nil {
30 - log.Error("failed to decode empty key constant")
31 - os.Exit(1)
32 - }
33 - emptyKey = e
34 -}
35 -
36 -const (
37 - linkRecursive = "recursive"
38 - linkDirect = "direct"
39 - linkIndirect = "indirect"
40 - linkInternal = "internal"
41 - linkNotPinned = "not pinned"
42 - linkAny = "any"
43 - linkAll = "all"
44 -)
45 -
46 -// Mode allows to specify different types of pin (recursive, direct etc.).
47 -// See the Pin Modes constants for a full list.
48 -type Mode int
49 -
50 -// Pin Modes
51 -const (
52 - // Recursive pins pin the target cids along with any reachable children.
53 - Recursive Mode = iota
54 -
55 - // Direct pins pin just the target cid.
56 - Direct
57 -
58 - // Indirect pins are cids who have some ancestor pinned recursively.
59 - Indirect
60 -
61 - // Internal pins are cids used to keep the internal state of the pinner.
62 - Internal
63 -
64 - // NotPinned
65 - NotPinned
66 -
67 - // Any refers to any pinned cid
68 - Any
69 -)
70 -
71 -// ModeToString returns a human-readable name for the Mode.
72 -func ModeToString(mode Mode) (string, bool) {
73 - m := map[Mode]string{
74 - Recursive: linkRecursive,
75 - Direct: linkDirect,
76 - Indirect: linkIndirect,
77 - Internal: linkInternal,
78 - NotPinned: linkNotPinned,
79 - Any: linkAny,
80 - }
81 - s, ok := m[mode]
82 - return s, ok
83 -}
84 -
85 -// StringToMode parses the result of ModeToString() back to a Mode.
86 -// It returns a boolean which is set to false if the mode is unknown.
87 -func StringToMode(s string) (Mode, bool) {
88 - m := map[string]Mode{
89 - linkRecursive: Recursive,
90 - linkDirect: Direct,
91 - linkIndirect: Indirect,
92 - linkInternal: Internal,
93 - linkNotPinned: NotPinned,
94 - linkAny: Any,
95 - linkAll: Any, // "all" and "any" means the same thing
96 - }
97 - mode, ok := m[s]
98 - return mode, ok
99 -}
100 -
101 -// A Pinner provides the necessary methods to keep track of Nodes which are
102 -// to be kept locally, according to a pin mode. In practice, a Pinner is in
103 -// in charge of keeping the list of items from the local storage that should
104 -// not be garbage-collected.
105 -type Pinner interface {
106 - // IsPinned returns whether or not the given cid is pinned
107 - // and an explanation of why its pinned
108 - IsPinned(ctx context.Context, c cid.Cid) (string, bool, error)
109 -
110 - // IsPinnedWithType returns whether or not the given cid is pinned with the
111 - // given pin type, as well as returning the type of pin its pinned with.
112 - IsPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error)
113 -
114 - // Pin the given node, optionally recursively.
115 - Pin(ctx context.Context, node ipld.Node, recursive bool) error
116 -
117 - // Unpin the given cid. If recursive is true, removes either a recursive or
118 - // a direct pin. If recursive is false, only removes a direct pin.
119 - Unpin(ctx context.Context, cid cid.Cid, recursive bool) error
120 -
121 - // Update updates a recursive pin from one cid to another
122 - // this is more efficient than simply pinning the new one and unpinning the
123 - // old one
124 - Update(ctx context.Context, from, to cid.Cid, unpin bool) error
125 -
126 - // Check if a set of keys are pinned, more efficient than
127 - // calling IsPinned for each key
128 - CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]Pinned, error)
129 -
130 - // PinWithMode is for manually editing the pin structure. Use with
131 - // care! If used improperly, garbage collection may not be
132 - // successful.
133 - PinWithMode(cid.Cid, Mode)
134 -
135 - // RemovePinWithMode is for manually editing the pin structure.
136 - // Use with care! If used improperly, garbage collection may not
137 - // be successful.
138 - RemovePinWithMode(cid.Cid, Mode)
139 -
140 - // Flush writes the pin state to the backing datastore
141 - Flush(ctx context.Context) error
142 -
143 - // DirectKeys returns all directly pinned cids
144 - DirectKeys(ctx context.Context) ([]cid.Cid, error)
145 -
146 - // DirectKeys returns all recursively pinned cids
147 - RecursiveKeys(ctx context.Context) ([]cid.Cid, error)
148 -
149 - // InternalPins returns all cids kept pinned for the internal state of the
150 - // pinner
151 - InternalPins(ctx context.Context) ([]cid.Cid, error)
152 -}
153 -
154 -// Pinned represents CID which has been pinned with a pinning strategy.
155 -// The Via field allows to identify the pinning parent of this CID, in the
156 -// case that the item is not pinned directly (but rather pinned recursively
157 -// by some ascendant).
158 -type Pinned struct {
159 - Key cid.Cid
160 - Mode Mode
161 - Via cid.Cid
162 -}
163 -
164 -// Pinned returns whether or not the given cid is pinned
165 -func (p Pinned) Pinned() bool {
166 - return p.Mode != NotPinned
167 -}
168 -
169 -// String Returns pin status as string
170 -func (p Pinned) String() string {
171 - switch p.Mode {
172 - case NotPinned:
173 - return "not pinned"
174 - case Indirect:
175 - return fmt.Sprintf("pinned via %s", p.Via)
176 - default:
177 - modeStr, _ := ModeToString(p.Mode)
178 - return fmt.Sprintf("pinned: %s", modeStr)
179 - }
180 -}
181 -
182 -// pinner implements the Pinner interface
183 -type pinner struct {
184 - lock sync.RWMutex
185 - recursePin *cid.Set
186 - directPin *cid.Set
187 -
188 - // Track the keys used for storing the pinning state, so gc does
189 - // not delete them.
190 - internalPin *cid.Set
191 - dserv ipld.DAGService
192 - internal ipld.DAGService // dagservice used to store internal objects
193 - dstore ds.Datastore
194 -}
195 -
196 -// NewPinner creates a new pinner using the given datastore as a backend
197 -func NewPinner(dstore ds.Datastore, serv, internal ipld.DAGService) Pinner {
198 -
199 - rcset := cid.NewSet()
200 - dirset := cid.NewSet()
201 -
202 - return &pinner{
203 - recursePin: rcset,
204 - directPin: dirset,
205 - dserv: serv,
206 - dstore: dstore,
207 - internal: internal,
208 - internalPin: cid.NewSet(),
209 - }
210 -}
211 -
212 -// Pin the given node, optionally recursive
213 -func (p *pinner) Pin(ctx context.Context, node ipld.Node, recurse bool) error {
214 - err := p.dserv.Add(ctx, node)
215 - if err != nil {
216 - return err
217 - }
218 -
219 - c := node.Cid()
220 -
221 - p.lock.Lock()
222 - defer p.lock.Unlock()
223 -
224 - if recurse {
225 - if p.recursePin.Has(c) {
226 - return nil
227 - }
228 -
229 - p.lock.Unlock()
230 - // temporary unlock to fetch the entire graph
231 - err := mdag.FetchGraph(ctx, c, p.dserv)
232 - p.lock.Lock()
233 - if err != nil {
234 - return err
235 - }
236 -
237 - if p.recursePin.Has(c) {
238 - return nil
239 - }
240 -
241 - if p.directPin.Has(c) {
242 - p.directPin.Remove(c)
243 - }
244 -
245 - p.recursePin.Add(c)
246 - } else {
247 - if p.recursePin.Has(c) {
248 - return fmt.Errorf("%s already pinned recursively", c.String())
249 - }
250 -
251 - p.directPin.Add(c)
252 - }
253 - return nil
254 -}
255 -
256 -// ErrNotPinned is returned when trying to unpin items which are not pinned.
257 -var ErrNotPinned = fmt.Errorf("not pinned or pinned indirectly")
258 -
259 -// Unpin a given key
260 -func (p *pinner) Unpin(ctx context.Context, c cid.Cid, recursive bool) error {
261 - p.lock.Lock()
262 - defer p.lock.Unlock()
263 - if p.recursePin.Has(c) {
264 - if !recursive {
265 - return fmt.Errorf("%s is pinned recursively", c)
266 - }
267 - p.recursePin.Remove(c)
268 - return nil
269 - }
270 - if p.directPin.Has(c) {
271 - p.directPin.Remove(c)
272 - return nil
273 - }
274 - return ErrNotPinned
275 -}
276 -
277 -func (p *pinner) isInternalPin(c cid.Cid) bool {
278 - return p.internalPin.Has(c)
279 -}
280 -
281 -// IsPinned returns whether or not the given key is pinned
282 -// and an explanation of why its pinned
283 -func (p *pinner) IsPinned(ctx context.Context, c cid.Cid) (string, bool, error) {
284 - p.lock.RLock()
285 - defer p.lock.RUnlock()
286 - return p.isPinnedWithType(ctx, c, Any)
287 -}
288 -
289 -// IsPinnedWithType returns whether or not the given cid is pinned with the
290 -// given pin type, as well as returning the type of pin its pinned with.
291 -func (p *pinner) IsPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) {
292 - p.lock.RLock()
293 - defer p.lock.RUnlock()
294 - return p.isPinnedWithType(ctx, c, mode)
295 -}
296 -
297 -// isPinnedWithType is the implementation of IsPinnedWithType that does not lock.
298 -// intended for use by other pinned methods that already take locks
299 -func (p *pinner) isPinnedWithType(ctx context.Context, c cid.Cid, mode Mode) (string, bool, error) {
300 - switch mode {
301 - case Any, Direct, Indirect, Recursive, Internal:
302 - default:
303 - err := fmt.Errorf("invalid Pin Mode '%d', must be one of {%d, %d, %d, %d, %d}",
304 - mode, Direct, Indirect, Recursive, Internal, Any)
305 - return "", false, err
306 - }
307 - if (mode == Recursive || mode == Any) && p.recursePin.Has(c) {
308 - return linkRecursive, true, nil
309 - }
310 - if mode == Recursive {
311 - return "", false, nil
312 - }
313 -
314 - if (mode == Direct || mode == Any) && p.directPin.Has(c) {
315 - return linkDirect, true, nil
316 - }
317 - if mode == Direct {
318 - return "", false, nil
319 - }
320 -
321 - if (mode == Internal || mode == Any) && p.isInternalPin(c) {
322 - return linkInternal, true, nil
323 - }
324 - if mode == Internal {
325 - return "", false, nil
326 - }
327 -
328 - // Default is Indirect
329 - visitedSet := cid.NewSet()
330 - for _, rc := range p.recursePin.Keys() {
331 - has, err := hasChild(ctx, p.dserv, rc, c, visitedSet.Visit)
332 - if err != nil {
333 - return "", false, err
334 - }
335 - if has {
336 - return rc.String(), true, nil
337 - }
338 - }
339 - return "", false, nil
340 -}
341 -
342 -// CheckIfPinned Checks if a set of keys are pinned, more efficient than
343 -// calling IsPinned for each key, returns the pinned status of cid(s)
344 -func (p *pinner) CheckIfPinned(ctx context.Context, cids ...cid.Cid) ([]Pinned, error) {
345 - p.lock.RLock()
346 - defer p.lock.RUnlock()
347 - pinned := make([]Pinned, 0, len(cids))
348 - toCheck := cid.NewSet()
349 -
350 - // First check for non-Indirect pins directly
351 - for _, c := range cids {
352 - if p.recursePin.Has(c) {
353 - pinned = append(pinned, Pinned{Key: c, Mode: Recursive})
354 - } else if p.directPin.Has(c) {
355 - pinned = append(pinned, Pinned{Key: c, Mode: Direct})
356 - } else if p.isInternalPin(c) {
357 - pinned = append(pinned, Pinned{Key: c, Mode: Internal})
358 - } else {
359 - toCheck.Add(c)
360 - }
361 - }
362 -
363 - // Now walk all recursive pins to check for indirect pins
364 - var checkChildren func(cid.Cid, cid.Cid) error
365 - checkChildren = func(rk, parentKey cid.Cid) error {
366 - links, err := ipld.GetLinks(ctx, p.dserv, parentKey)
367 - if err != nil {
368 - return err
369 - }
370 - for _, lnk := range links {
371 - c := lnk.Cid
372 -
373 - if toCheck.Has(c) {
374 - pinned = append(pinned,
375 - Pinned{Key: c, Mode: Indirect, Via: rk})
376 - toCheck.Remove(c)
377 - }
378 -
379 - err := checkChildren(rk, c)
380 - if err != nil {
381 - return err
382 - }
383 -
384 - if toCheck.Len() == 0 {
385 - return nil
386 - }
387 - }
388 - return nil
389 - }
390 -
391 - for _, rk := range p.recursePin.Keys() {
392 - err := checkChildren(rk, rk)
393 - if err != nil {
394 - return nil, err
395 - }
396 - if toCheck.Len() == 0 {
397 - break
398 - }
399 - }
400 -
401 - // Anything left in toCheck is not pinned
402 - for _, k := range toCheck.Keys() {
403 - pinned = append(pinned, Pinned{Key: k, Mode: NotPinned})
404 - }
405 -
406 - return pinned, nil
407 -}
408 -
409 -// RemovePinWithMode is for manually editing the pin structure.
410 -// Use with care! If used improperly, garbage collection may not
411 -// be successful.
412 -func (p *pinner) RemovePinWithMode(c cid.Cid, mode Mode) {
413 - p.lock.Lock()
414 - defer p.lock.Unlock()
415 - switch mode {
416 - case Direct:
417 - p.directPin.Remove(c)
418 - case Recursive:
419 - p.recursePin.Remove(c)
420 - default:
421 - // programmer error, panic OK
422 - panic("unrecognized pin type")
423 - }
424 -}
425 -
426 -func cidSetWithValues(cids []cid.Cid) *cid.Set {
427 - out := cid.NewSet()
428 - for _, c := range cids {
429 - out.Add(c)
430 - }
431 - return out
432 -}
433 -
434 -// LoadPinner loads a pinner and its keysets from the given datastore
435 -func LoadPinner(d ds.Datastore, dserv, internal ipld.DAGService) (Pinner, error) {
436 - p := new(pinner)
437 -
438 - rootKey, err := d.Get(pinDatastoreKey)
439 - if err != nil {
440 - return nil, fmt.Errorf("cannot load pin state: %v", err)
441 - }
442 - rootCid, err := cid.Cast(rootKey)
443 - if err != nil {
444 - return nil, err
445 - }
446 -
447 - ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5)
448 - defer cancel()
449 -
450 - root, err := internal.Get(ctx, rootCid)
451 - if err != nil {
452 - return nil, fmt.Errorf("cannot find pinning root object: %v", err)
453 - }
454 -
455 - rootpb, ok := root.(*mdag.ProtoNode)
456 - if !ok {
457 - return nil, mdag.ErrNotProtobuf
458 - }
459 -
460 - internalset := cid.NewSet()
461 - internalset.Add(rootCid)
462 - recordInternal := internalset.Add
463 -
464 - { // load recursive set
465 - recurseKeys, err := loadSet(ctx, internal, rootpb, linkRecursive, recordInternal)
466 - if err != nil {
467 - return nil, fmt.Errorf("cannot load recursive pins: %v", err)
468 - }
469 - p.recursePin = cidSetWithValues(recurseKeys)
470 - }
471 -
472 - { // load direct set
473 - directKeys, err := loadSet(ctx, internal, rootpb, linkDirect, recordInternal)
474 - if err != nil {
475 - return nil, fmt.Errorf("cannot load direct pins: %v", err)
476 - }
477 - p.directPin = cidSetWithValues(directKeys)
478 - }
479 -
480 - p.internalPin = internalset
481 -
482 - // assign services
483 - p.dserv = dserv
484 - p.dstore = d
485 - p.internal = internal
486 -
487 - return p, nil
488 -}
489 -
490 -// DirectKeys returns a slice containing the directly pinned keys
491 -func (p *pinner) DirectKeys(ctx context.Context) ([]cid.Cid, error) {
492 - p.lock.RLock()
493 - defer p.lock.RUnlock()
494 -
495 - return p.directPin.Keys(), nil
496 -}
497 -
498 -// RecursiveKeys returns a slice containing the recursively pinned keys
499 -func (p *pinner) RecursiveKeys(ctx context.Context) ([]cid.Cid, error) {
500 - p.lock.RLock()
501 - defer p.lock.RUnlock()
502 -
503 - return p.recursePin.Keys(), nil
504 -}
505 -
506 -// Update updates a recursive pin from one cid to another
507 -// this is more efficient than simply pinning the new one and unpinning the
508 -// old one
509 -func (p *pinner) Update(ctx context.Context, from, to cid.Cid, unpin bool) error {
510 - if from == to {
511 - // Nothing to do. Don't remove this check or we'll end up
512 - // _removing_ the pin.
513 - //
514 - // See #6648
515 - return nil
516 - }
517 -
518 - p.lock.Lock()
519 - defer p.lock.Unlock()
520 -
521 - if !p.recursePin.Has(from) {
522 - return fmt.Errorf("'from' cid was not recursively pinned already")
523 - }
524 -
525 - err := dagutils.DiffEnumerate(ctx, p.dserv, from, to)
526 - if err != nil {
527 - return err
528 - }
529 -
530 - p.recursePin.Add(to)
531 - if unpin {
532 - p.recursePin.Remove(from)
533 - }
534 - return nil
535 -}
536 -
537 -// Flush encodes and writes pinner keysets to the datastore
538 -func (p *pinner) Flush(ctx context.Context) error {
539 - p.lock.Lock()
540 - defer p.lock.Unlock()
541 -
542 - internalset := cid.NewSet()
543 - recordInternal := internalset.Add
544 -
545 - root := &mdag.ProtoNode{}
546 - {
547 - n, err := storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal)
548 - if err != nil {
549 - return err
550 - }
551 - if err := root.AddNodeLink(linkDirect, n); err != nil {
552 - return err
553 - }
554 - }
555 -
556 - {
557 - n, err := storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal)
558 - if err != nil {
559 - return err
560 - }
561 - if err := root.AddNodeLink(linkRecursive, n); err != nil {
562 - return err
563 - }
564 - }
565 -
566 - // add the empty node, its referenced by the pin sets but never created
567 - err := p.internal.Add(ctx, new(mdag.ProtoNode))
568 - if err != nil {
569 - return err
570 - }
571 -
572 - err = p.internal.Add(ctx, root)
573 - if err != nil {
574 - return err
575 - }
576 -
577 - k := root.Cid()
578 -
579 - internalset.Add(k)
580 - if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil {
581 - return fmt.Errorf("cannot store pin state: %v", err)
582 - }
583 - p.internalPin = internalset
584 - return nil
585 -}
586 -
587 -// InternalPins returns all cids kept pinned for the internal state of the
588 -// pinner
589 -func (p *pinner) InternalPins(ctx context.Context) ([]cid.Cid, error) {
590 - p.lock.Lock()
591 - defer p.lock.Unlock()
592 - var out []cid.Cid
593 - out = append(out, p.internalPin.Keys()...)
594 - return out, nil
595 -}
596 -
597 -// PinWithMode allows the user to have fine grained control over pin
598 -// counts
599 -func (p *pinner) PinWithMode(c cid.Cid, mode Mode) {
600 - p.lock.Lock()
601 - defer p.lock.Unlock()
602 - switch mode {
603 - case Recursive:
604 - p.recursePin.Add(c)
605 - case Direct:
606 - p.directPin.Add(c)
607 - }
608 -}
609 -
610 -// hasChild recursively looks for a Cid among the children of a root Cid.
611 -// The visit function can be used to shortcut already-visited branches.
612 -func hasChild(ctx context.Context, ng ipld.NodeGetter, root cid.Cid, child cid.Cid, visit func(cid.Cid) bool) (bool, error) {
613 - links, err := ipld.GetLinks(ctx, ng, root)
614 - if err != nil {
615 - return false, err
616 - }
617 - for _, lnk := range links {
618 - c := lnk.Cid
619 - if lnk.Cid.Equals(child) {
620 - return true, nil
621 - }
622 - if visit(c) {
623 - has, err := hasChild(ctx, ng, c, child, visit)
624 - if err != nil {
625 - return false, err
626 - }
627 -
628 - if has {
629 - return has, nil
630 - }
631 - }
632 - }
633 - return false, nil
634 -}
pin/pin_test.go deleted
-416
@@ -1,416 +0,0 @@
1 -package pin
2 -
3 -import (
4 - "context"
5 - "io"
6 - "testing"
7 - "time"
8 -
9 - bs "github.com/ipfs/go-blockservice"
10 - mdag "github.com/ipfs/go-merkledag"
11 -
12 - cid "github.com/ipfs/go-cid"
13 - ds "github.com/ipfs/go-datastore"
14 - dssync "github.com/ipfs/go-datastore/sync"
15 - blockstore "github.com/ipfs/go-ipfs-blockstore"
16 - offline "github.com/ipfs/go-ipfs-exchange-offline"
17 - util "github.com/ipfs/go-ipfs-util"
18 -)
19 -
20 -var rand = util.NewTimeSeededRand()
21 -
22 -func randNode() (*mdag.ProtoNode, cid.Cid) {
23 - nd := new(mdag.ProtoNode)
24 - nd.SetData(make([]byte, 32))
25 - _, err := io.ReadFull(rand, nd.Data())
26 - if err != nil {
27 - panic(err)
28 - }
29 - k := nd.Cid()
30 - return nd, k
31 -}
32 -
33 -func assertPinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) {
34 - _, pinned, err := p.IsPinned(context.Background(), c)
35 - if err != nil {
36 - t.Fatal(err)
37 - }
38 -
39 - if !pinned {
40 - t.Fatal(failmsg)
41 - }
42 -}
43 -
44 -func assertUnpinned(t *testing.T, p Pinner, c cid.Cid, failmsg string) {
45 - _, pinned, err := p.IsPinned(context.Background(), c)
46 - if err != nil {
47 - t.Fatal(err)
48 - }
49 -
50 - if pinned {
51 - t.Fatal(failmsg)
52 - }
53 -}
54 -
55 -func TestPinnerBasic(t *testing.T) {
56 - ctx := context.Background()
57 -
58 - dstore := dssync.MutexWrap(ds.NewMapDatastore())
59 - bstore := blockstore.NewBlockstore(dstore)
60 - bserv := bs.New(bstore, offline.Exchange(bstore))
61 -
62 - dserv := mdag.NewDAGService(bserv)
63 -
64 - // TODO does pinner need to share datastore with blockservice?
65 - p := NewPinner(dstore, dserv, dserv)
66 -
67 - a, ak := randNode()
68 - err := dserv.Add(ctx, a)
69 - if err != nil {
70 - t.Fatal(err)
71 - }
72 -
73 - // Pin A{}
74 - err = p.Pin(ctx, a, false)
75 - if err != nil {
76 - t.Fatal(err)
77 - }
78 -
79 - assertPinned(t, p, ak, "Failed to find key")
80 -
81 - // create new node c, to be indirectly pinned through b
82 - c, _ := randNode()
83 - err = dserv.Add(ctx, c)
84 - if err != nil {
85 - t.Fatal(err)
86 - }
87 - ck := c.Cid()
88 -
89 - // Create new node b, to be parent to a and c
90 - b, _ := randNode()
91 - err = b.AddNodeLink("child", a)
92 - if err != nil {
93 - t.Fatal(err)
94 - }
95 -
96 - err = b.AddNodeLink("otherchild", c)
97 - if err != nil {
98 - t.Fatal(err)
99 - }
100 -
101 - err = dserv.Add(ctx, b)
102 - if err != nil {
103 - t.Fatal(err)
104 - }
105 - bk := b.Cid()
106 -
107 - // recursively pin B{A,C}
108 - err = p.Pin(ctx, b, true)
109 - if err != nil {
110 - t.Fatal(err)
111 - }
112 -
113 - assertPinned(t, p, ck, "child of recursively pinned node not found")
114 -
115 - assertPinned(t, p, bk, "Recursively pinned node not found..")
116 -
117 - d, _ := randNode()
118 - _ = d.AddNodeLink("a", a)
119 - _ = d.AddNodeLink("c", c)
120 -
121 - e, _ := randNode()
122 - _ = d.AddNodeLink("e", e)
123 -
124 - // Must be in dagserv for unpin to work
125 - err = dserv.Add(ctx, e)
126 - if err != nil {
127 - t.Fatal(err)
128 - }
129 - err = dserv.Add(ctx, d)
130 - if err != nil {
131 - t.Fatal(err)
132 - }
133 -
134 - // Add D{A,C,E}
135 - err = p.Pin(ctx, d, true)
136 - if err != nil {
137 - t.Fatal(err)
138 - }
139 -
140 - dk := d.Cid()
141 - assertPinned(t, p, dk, "pinned node not found.")
142 -
143 - // Test recursive unpin
144 - err = p.Unpin(ctx, dk, true)
145 - if err != nil {
146 - t.Fatal(err)
147 - }
148 -
149 - err = p.Flush(ctx)
150 - if err != nil {
151 - t.Fatal(err)
152 - }
153 -
154 - np, err := LoadPinner(dstore, dserv, dserv)
155 - if err != nil {
156 - t.Fatal(err)
157 - }
158 -
159 - // Test directly pinned
160 - assertPinned(t, np, ak, "Could not find pinned node!")
161 -
162 - // Test recursively pinned
163 - assertPinned(t, np, bk, "could not find recursively pinned node")
164 -}
165 -
166 -func TestIsPinnedLookup(t *testing.T) {
167 - // We are going to test that lookups work in pins which share
168 - // the same branches. For that we will construct this tree:
169 - //
170 - // A5->A4->A3->A2->A1->A0
171 - // / /
172 - // B------- /
173 - // \ /
174 - // C---------------
175 - //
176 - // We will ensure that IsPinned works for all objects both when they
177 - // are pinned and once they have been unpinned.
178 - aBranchLen := 6
179 - if aBranchLen < 3 {
180 - t.Fatal("set aBranchLen to at least 3")
181 - }
182 -
183 - ctx := context.Background()
184 - dstore := dssync.MutexWrap(ds.NewMapDatastore())
185 - bstore := blockstore.NewBlockstore(dstore)
186 - bserv := bs.New(bstore, offline.Exchange(bstore))
187 -
188 - dserv := mdag.NewDAGService(bserv)
189 -
190 - // TODO does pinner need to share datastore with blockservice?
191 - p := NewPinner(dstore, dserv, dserv)
192 -
193 - aNodes := make([]*mdag.ProtoNode, aBranchLen)
194 - aKeys := make([]cid.Cid, aBranchLen)
195 - for i := 0; i < aBranchLen; i++ {
196 - a, _ := randNode()
197 - if i >= 1 {
198 - err := a.AddNodeLink("child", aNodes[i-1])
199 - if err != nil {
200 - t.Fatal(err)
201 - }
202 - }
203 -
204 - err := dserv.Add(ctx, a)
205 - if err != nil {
206 - t.Fatal(err)
207 - }
208 - //t.Logf("a[%d] is %s", i, ak)
209 - aNodes[i] = a
210 - aKeys[i] = a.Cid()
211 - }
212 -
213 - // Pin A5 recursively
214 - if err := p.Pin(ctx, aNodes[aBranchLen-1], true); err != nil {
215 - t.Fatal(err)
216 - }
217 -
218 - // Create node B and add A3 as child
219 - b, _ := randNode()
220 - if err := b.AddNodeLink("mychild", aNodes[3]); err != nil {
221 - t.Fatal(err)
222 - }
223 -
224 - // Create C node
225 - c, _ := randNode()
226 - // Add A0 as child of C
227 - if err := c.AddNodeLink("child", aNodes[0]); err != nil {
228 - t.Fatal(err)
229 - }
230 -
231 - // Add C
232 - err := dserv.Add(ctx, c)
233 - if err != nil {
234 - t.Fatal(err)
235 - }
236 - ck := c.Cid()
237 - //t.Logf("C is %s", ck)
238 -
239 - // Add C to B and Add B
240 - if err := b.AddNodeLink("myotherchild", c); err != nil {
241 - t.Fatal(err)
242 - }
243 - err = dserv.Add(ctx, b)
244 - if err != nil {
245 - t.Fatal(err)
246 - }
247 - bk := b.Cid()
248 - //t.Logf("B is %s", bk)
249 -
250 - // Pin C recursively
251 -
252 - if err := p.Pin(ctx, c, true); err != nil {
253 - t.Fatal(err)
254 - }
255 -
256 - // Pin B recursively
257 -
258 - if err := p.Pin(ctx, b, true); err != nil {
259 - t.Fatal(err)
260 - }
261 -
262 - assertPinned(t, p, aKeys[0], "A0 should be pinned")
263 - assertPinned(t, p, aKeys[1], "A1 should be pinned")
264 - assertPinned(t, p, ck, "C should be pinned")
265 - assertPinned(t, p, bk, "B should be pinned")
266 -
267 - // Unpin A5 recursively
268 - if err := p.Unpin(ctx, aKeys[5], true); err != nil {
269 - t.Fatal(err)
270 - }
271 -
272 - assertPinned(t, p, aKeys[0], "A0 should still be pinned through B")
273 - assertUnpinned(t, p, aKeys[4], "A4 should be unpinned")
274 -
275 - // Unpin B recursively
276 - if err := p.Unpin(ctx, bk, true); err != nil {
277 - t.Fatal(err)
278 - }
279 - assertUnpinned(t, p, bk, "B should be unpinned")
280 - assertUnpinned(t, p, aKeys[1], "A1 should be unpinned")
281 - assertPinned(t, p, aKeys[0], "A0 should still be pinned through C")
282 -}
283 -
284 -func TestDuplicateSemantics(t *testing.T) {
285 - ctx := context.Background()
286 - dstore := dssync.MutexWrap(ds.NewMapDatastore())
287 - bstore := blockstore.NewBlockstore(dstore)
288 - bserv := bs.New(bstore, offline.Exchange(bstore))
289 -
290 - dserv := mdag.NewDAGService(bserv)
291 -
292 - // TODO does pinner need to share datastore with blockservice?
293 - p := NewPinner(dstore, dserv, dserv)
294 -
295 - a, _ := randNode()
296 - err := dserv.Add(ctx, a)
297 - if err != nil {
298 - t.Fatal(err)
299 - }
300 -
301 - // pin is recursively
302 - err = p.Pin(ctx, a, true)
303 - if err != nil {
304 - t.Fatal(err)
305 - }
306 -
307 - // pinning directly should fail
308 - err = p.Pin(ctx, a, false)
309 - if err == nil {
310 - t.Fatal("expected direct pin to fail")
311 - }
312 -
313 - // pinning recursively again should succeed
314 - err = p.Pin(ctx, a, true)
315 - if err != nil {
316 - t.Fatal(err)
317 - }
318 -}
319 -
320 -func TestFlush(t *testing.T) {
321 - dstore := dssync.MutexWrap(ds.NewMapDatastore())
322 - bstore := blockstore.NewBlockstore(dstore)
323 - bserv := bs.New(bstore, offline.Exchange(bstore))
324 -
325 - dserv := mdag.NewDAGService(bserv)
326 - p := NewPinner(dstore, dserv, dserv)
327 - _, k := randNode()
328 -
329 - p.PinWithMode(k, Recursive)
330 - if err := p.Flush(context.Background()); err != nil {
331 - t.Fatal(err)
332 - }
333 - assertPinned(t, p, k, "expected key to still be pinned")
334 -}
335 -
336 -func TestPinRecursiveFail(t *testing.T) {
337 - ctx := context.Background()
338 - dstore := dssync.MutexWrap(ds.NewMapDatastore())
339 - bstore := blockstore.NewBlockstore(dstore)
340 - bserv := bs.New(bstore, offline.Exchange(bstore))
341 - dserv := mdag.NewDAGService(bserv)
342 -
343 - p := NewPinner(dstore, dserv, dserv)
344 -
345 - a, _ := randNode()
346 - b, _ := randNode()
347 - err := a.AddNodeLink("child", b)
348 - if err != nil {
349 - t.Fatal(err)
350 - }
351 -
352 - // NOTE: This isnt a time based test, we expect the pin to fail
353 - mctx, cancel := context.WithTimeout(ctx, time.Millisecond)
354 - defer cancel()
355 -
356 - err = p.Pin(mctx, a, true)
357 - if err == nil {
358 - t.Fatal("should have failed to pin here")
359 - }
360 -
361 - err = dserv.Add(ctx, b)
362 - if err != nil {
363 - t.Fatal(err)
364 - }
365 -
366 - err = dserv.Add(ctx, a)
367 - if err != nil {
368 - t.Fatal(err)
369 - }
370 -
371 - // this one is time based... but shouldnt cause any issues
372 - mctx, cancel = context.WithTimeout(ctx, time.Second)
373 - defer cancel()
374 - err = p.Pin(mctx, a, true)
375 - if err != nil {
376 - t.Fatal(err)
377 - }
378 -}
379 -
380 -func TestPinUpdate(t *testing.T) {
381 - ctx := context.Background()
382 -
383 - dstore := dssync.MutexWrap(ds.NewMapDatastore())
384 - bstore := blockstore.NewBlockstore(dstore)
385 - bserv := bs.New(bstore, offline.Exchange(bstore))
386 -
387 - dserv := mdag.NewDAGService(bserv)
388 - p := NewPinner(dstore, dserv, dserv)
389 - n1, c1 := randNode()
390 - n2, c2 := randNode()
391 -
392 - if err := dserv.Add(ctx, n1); err != nil {
393 - t.Fatal(err)
394 - }
395 - if err := dserv.Add(ctx, n2); err != nil {
396 - t.Fatal(err)
397 - }
398 -
399 - if err := p.Pin(ctx, n1, true); err != nil {
400 - t.Fatal(err)
401 - }
402 -
403 - if err := p.Update(ctx, c1, c2, true); err != nil {
404 - t.Fatal(err)
405 - }
406 -
407 - assertPinned(t, p, c2, "c2 should be pinned now")
408 - assertUnpinned(t, p, c1, "c1 should no longer be pinned")
409 -
410 - if err := p.Update(ctx, c2, c1, false); err != nil {
411 - t.Fatal(err)
412 - }
413 -
414 - assertPinned(t, p, c2, "c2 should be pinned still")
415 - assertPinned(t, p, c1, "c1 should be pinned now")
416 -}
pin/set.go deleted
-297
@@ -1,297 +0,0 @@
1 -package pin
2 -
3 -import (
4 - "bytes"
5 - "context"
6 - "encoding/binary"
7 - "errors"
8 - "fmt"
9 - "hash/fnv"
10 - "sort"
11 -
12 - "github.com/ipfs/go-ipfs/pin/internal/pb"
13 - "github.com/ipfs/go-merkledag"
14 -
15 - "github.com/gogo/protobuf/proto"
16 - cid "github.com/ipfs/go-cid"
17 - ipld "github.com/ipfs/go-ipld-format"
18 -)
19 -
20 -const (
21 - // defaultFanout specifies the default number of fan-out links per layer
22 - defaultFanout = 256
23 -
24 - // maxItems is the maximum number of items that will fit in a single bucket
25 - maxItems = 8192
26 -)
27 -
28 -func hash(seed uint32, c cid.Cid) uint32 {
29 - var buf [4]byte
30 - binary.LittleEndian.PutUint32(buf[:], seed)
31 - h := fnv.New32a()
32 - _, _ = h.Write(buf[:])
33 - _, _ = h.Write(c.Bytes())
34 - return h.Sum32()
35 -}
36 -
37 -type itemIterator func() (c cid.Cid, ok bool)
38 -
39 -type keyObserver func(cid.Cid)
40 -
41 -type sortByHash struct {
42 - links []*ipld.Link
43 -}
44 -
45 -func (s sortByHash) Len() int {
46 - return len(s.links)
47 -}
48 -
49 -func (s sortByHash) Less(a, b int) bool {
50 - return bytes.Compare(s.links[a].Cid.Bytes(), s.links[b].Cid.Bytes()) == -1
51 -}
52 -
53 -func (s sortByHash) Swap(a, b int) {
54 - s.links[a], s.links[b] = s.links[b], s.links[a]
55 -}
56 -
57 -func storeItems(ctx context.Context, dag ipld.DAGService, estimatedLen uint64, depth uint32, iter itemIterator, internalKeys keyObserver) (*merkledag.ProtoNode, error) {
58 - links := make([]*ipld.Link, 0, defaultFanout+maxItems)
59 - for i := 0; i < defaultFanout; i++ {
60 - links = append(links, &ipld.Link{Cid: emptyKey})
61 - }
62 -
63 - // add emptyKey to our set of internal pinset objects
64 - n := &merkledag.ProtoNode{}
65 - n.SetLinks(links)
66 -
67 - internalKeys(emptyKey)
68 -
69 - hdr := &pb.Set{
70 - Version: 1,
71 - Fanout: defaultFanout,
72 - Seed: depth,
73 - }
74 - if err := writeHdr(n, hdr); err != nil {
75 - return nil, err
76 - }
77 -
78 - if estimatedLen < maxItems {
79 - // it'll probably fit
80 - links := n.Links()
81 - for i := 0; i < maxItems; i++ {
82 - k, ok := iter()
83 - if !ok {
84 - // all done
85 - break
86 - }
87 -
88 - links = append(links, &ipld.Link{Cid: k})
89 - }
90 -
91 - n.SetLinks(links)
92 -
93 - // sort by hash, also swap item Data
94 - s := sortByHash{
95 - links: n.Links()[defaultFanout:],
96 - }
97 - sort.Stable(s)
98 - }
99 -
100 - hashed := make([][]cid.Cid, defaultFanout)
101 - for {
102 - // This loop essentially enumerates every single item in the set
103 - // and maps them all into a set of buckets. Each bucket will be recursively
104 - // turned into its own sub-set, and so on down the chain. Each sub-set
105 - // gets added to the dagservice, and put into its place in a set nodes
106 - // links array.
107 - //
108 - // Previously, the bucket was selected by taking an int32 from the hash of
109 - // the input key + seed. This was erroneous as we would later be assigning
110 - // the created sub-sets into an array of length 256 by the modulus of the
111 - // int32 hash value with 256. This resulted in overwriting existing sub-sets
112 - // and losing pins. The fix (a few lines down from this comment), is to
113 - // map the hash value down to the 8 bit keyspace here while creating the
114 - // buckets. This way, we avoid any overlapping later on.
115 - k, ok := iter()
116 - if !ok {
117 - break
118 - }
119 - h := hash(depth, k) % defaultFanout
120 - hashed[h] = append(hashed[h], k)
121 - }
122 -
123 - for h, items := range hashed {
124 - if len(items) == 0 {
125 - // recursion base case
126 - continue
127 - }
128 -
129 - childIter := getCidListIterator(items)
130 -
131 - // recursively create a pinset from the items for this bucket index
132 - child, err := storeItems(ctx, dag, uint64(len(items)), depth+1, childIter, internalKeys)
133 - if err != nil {
134 - return nil, err
135 - }
136 -
137 - size, err := child.Size()
138 - if err != nil {
139 - return nil, err
140 - }
141 -
142 - err = dag.Add(ctx, child)
143 - if err != nil {
144 - return nil, err
145 - }
146 - childKey := child.Cid()
147 -
148 - internalKeys(childKey)
149 -
150 - // overwrite the 'empty key' in the existing links array
151 - n.Links()[h] = &ipld.Link{
152 - Cid: childKey,
153 - Size: size,
154 - }
155 - }
156 - return n, nil
157 -}
158 -
159 -func readHdr(n *merkledag.ProtoNode) (*pb.Set, error) {
160 - hdrLenRaw, consumed := binary.Uvarint(n.Data())
161 - if consumed <= 0 {
162 - return nil, errors.New("invalid Set header length")
163 - }
164 -
165 - pbdata := n.Data()[consumed:]
166 - if hdrLenRaw > uint64(len(pbdata)) {
167 - return nil, errors.New("impossibly large Set header length")
168 - }
169 - // as hdrLenRaw was <= an int, we now know it fits in an int
170 - hdrLen := int(hdrLenRaw)
171 - var hdr pb.Set
172 - if err := proto.Unmarshal(pbdata[:hdrLen], &hdr); err != nil {
173 - return nil, err
174 - }
175 -
176 - if v := hdr.GetVersion(); v != 1 {
177 - return nil, fmt.Errorf("unsupported Set version: %d", v)
178 - }
179 - if uint64(hdr.GetFanout()) > uint64(len(n.Links())) {
180 - return nil, errors.New("impossibly large Fanout")
181 - }
182 - return &hdr, nil
183 -}
184 -
185 -func writeHdr(n *merkledag.ProtoNode, hdr *pb.Set) error {
186 - hdrData, err := proto.Marshal(hdr)
187 - if err != nil {
188 - return err
189 - }
190 -
191 - // make enough space for the length prefix and the marshaled header data
192 - data := make([]byte, binary.MaxVarintLen64, binary.MaxVarintLen64+len(hdrData))
193 -
194 - // write the uvarint length of the header data
195 - uvarlen := binary.PutUvarint(data, uint64(len(hdrData)))
196 -
197 - // append the actual protobuf data *after* the length value we wrote
198 - data = append(data[:uvarlen], hdrData...)
199 -
200 - n.SetData(data)
201 - return nil
202 -}
203 -
204 -type walkerFunc func(idx int, link *ipld.Link) error
205 -
206 -func walkItems(ctx context.Context, dag ipld.DAGService, n *merkledag.ProtoNode, fn walkerFunc, children keyObserver) error {
207 - hdr, err := readHdr(n)
208 - if err != nil {
209 - return err
210 - }
211 - // readHdr guarantees fanout is a safe value
212 - fanout := hdr.GetFanout()
213 - for i, l := range n.Links()[fanout:] {
214 - if err := fn(i, l); err != nil {
215 - return err
216 - }
217 - }
218 - for _, l := range n.Links()[:fanout] {
219 - c := l.Cid
220 - children(c)
221 - if c.Equals(emptyKey) {
222 - continue
223 - }
224 - subtree, err := l.GetNode(ctx, dag)
225 - if err != nil {
226 - return err
227 - }
228 -
229 - stpb, ok := subtree.(*merkledag.ProtoNode)
230 - if !ok {
231 - return merkledag.ErrNotProtobuf
232 - }
233 -
234 - if err := walkItems(ctx, dag, stpb, fn, children); err != nil {
235 - return err
236 - }
237 - }
238 - return nil
239 -}
240 -
241 -func loadSet(ctx context.Context, dag ipld.DAGService, root *merkledag.ProtoNode, name string, internalKeys keyObserver) ([]cid.Cid, error) {
242 - l, err := root.GetNodeLink(name)
243 - if err != nil {
244 - return nil, err
245 - }
246 -
247 - lnkc := l.Cid
248 - internalKeys(lnkc)
249 -
250 - n, err := l.GetNode(ctx, dag)
251 - if err != nil {
252 - return nil, err
253 - }
254 -
255 - pbn, ok := n.(*merkledag.ProtoNode)
256 - if !ok {
257 - return nil, merkledag.ErrNotProtobuf
258 - }
259 -
260 - var res []cid.Cid
261 - walk := func(idx int, link *ipld.Link) error {
262 - res = append(res, link.Cid)
263 - return nil
264 - }
265 -
266 - if err := walkItems(ctx, dag, pbn, walk, internalKeys); err != nil {
267 - return nil, err
268 - }
269 - return res, nil
270 -}
271 -
272 -func getCidListIterator(cids []cid.Cid) itemIterator {
273 - return func() (c cid.Cid, ok bool) {
274 - if len(cids) == 0 {
275 - return cid.Cid{}, false
276 - }
277 -
278 - first := cids[0]
279 - cids = cids[1:]
280 - return first, true
281 - }
282 -}
283 -
284 -func storeSet(ctx context.Context, dag ipld.DAGService, cids []cid.Cid, internalKeys keyObserver) (*merkledag.ProtoNode, error) {
285 - iter := getCidListIterator(cids)
286 -
287 - n, err := storeItems(ctx, dag, uint64(len(cids)), 0, iter, internalKeys)
288 - if err != nil {
289 - return nil, err
290 - }
291 - err = dag.Add(ctx, n)
292 - if err != nil {
293 - return nil, err
294 - }
295 - internalKeys(n.Cid())
296 - return n, nil
297 -}
pin/set_test.go deleted
-101
@@ -1,101 +0,0 @@
1 -package pin
2 -
3 -import (
4 - "context"
5 - "encoding/binary"
6 - "testing"
7 -
8 - bserv "github.com/ipfs/go-blockservice"
9 - dag "github.com/ipfs/go-merkledag"
10 -
11 - cid "github.com/ipfs/go-cid"
12 - ds "github.com/ipfs/go-datastore"
13 - dsq "github.com/ipfs/go-datastore/query"
14 - blockstore "github.com/ipfs/go-ipfs-blockstore"
15 - offline "github.com/ipfs/go-ipfs-exchange-offline"
16 -)
17 -
18 -func ignoreCids(_ cid.Cid) {}
19 -
20 -func objCount(d ds.Datastore) int {
21 - q := dsq.Query{KeysOnly: true}
22 - res, err := d.Query(q)
23 - if err != nil {
24 - panic(err)
25 - }
26 -
27 - var count int
28 - for {
29 - _, ok := res.NextSync()
30 - if !ok {
31 - break
32 - }
33 -
34 - count++
35 - }
36 - return count
37 -}
38 -
39 -func TestSet(t *testing.T) {
40 - dst := ds.NewMapDatastore()
41 - bstore := blockstore.NewBlockstore(dst)
42 - ds := dag.NewDAGService(bserv.New(bstore, offline.Exchange(bstore)))
43 -
44 - // this value triggers the creation of a recursive shard.
45 - // If the recursive sharding is done improperly, this will result in
46 - // an infinite recursion and crash (OOM)
47 - limit := uint32((defaultFanout * maxItems) + 1)
48 -
49 - var inputs []cid.Cid
50 - buf := make([]byte, 4)
51 - for i := uint32(0); i < limit; i++ {
52 - binary.BigEndian.PutUint32(buf, i)
53 - c := dag.NewRawNode(buf).Cid()
54 - inputs = append(inputs, c)
55 - }
56 -
57 - _, err := storeSet(context.Background(), ds, inputs[:len(inputs)-1], ignoreCids)
58 - if err != nil {
59 - t.Fatal(err)
60 - }
61 -
62 - objs1 := objCount(dst)
63 -
64 - out, err := storeSet(context.Background(), ds, inputs, ignoreCids)
65 - if err != nil {
66 - t.Fatal(err)
67 - }
68 -
69 - objs2 := objCount(dst)
70 - if objs2-objs1 > 2 {
71 - t.Fatal("set sharding does not appear to be deterministic")
72 - }
73 -
74 - // weird wrapper node because loadSet expects us to pass an
75 - // object pointing to multiple named sets
76 - setroot := &dag.ProtoNode{}
77 - err = setroot.AddNodeLink("foo", out)
78 - if err != nil {
79 - t.Fatal(err)
80 - }
81 -
82 - outset, err := loadSet(context.Background(), ds, setroot, "foo", ignoreCids)
83 - if err != nil {
84 - t.Fatal(err)
85 - }
86 -
87 - if uint32(len(outset)) != limit {
88 - t.Fatal("got wrong number", len(outset), limit)
89 - }
90 -
91 - seen := cid.NewSet()
92 - for _, c := range outset {
93 - seen.Add(c)
94 - }
95 -
96 - for _, c := range inputs {
97 - if !seen.Has(c) {
98 - t.Fatalf("expected to have '%s', didnt find it", c)
99 - }
100 - }
101 -}
tar/format.go
+1 -1
@@ -8,8 +8,8 @@ import (
8 "io"
9 "strings"
10
11 - "github.com/ipfs/go-ipfs/dagutils"
11 dag "github.com/ipfs/go-merkledag"
12 + "github.com/ipfs/go-merkledag/dagutils"
13 path "github.com/ipfs/go-path"
14 importer "github.com/ipfs/go-unixfs/importer"
15 uio "github.com/ipfs/go-unixfs/io"