@cryptotaxi247 / kubo / commits / 815a031f5

Implement ipns republisher

This commit adds a very basic process that will periodically go through a list of given ids and republish the values for their ipns entries. License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>

Jeromy committed Sep 21, 2015 at 16:35 UTC 815a031f595bbe68d084c2666e0c353469bcb05a
8 files changed +356 -25
core/core.go
+25
@@ -51,6 +51,7 @@ import (
51 ipnsfs "github.com/ipfs/go-ipfs/ipnsfs"
52 merkledag "github.com/ipfs/go-ipfs/merkledag"
53 namesys "github.com/ipfs/go-ipfs/namesys"
54 + ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher"
55 path "github.com/ipfs/go-ipfs/path"
56 pin "github.com/ipfs/go-ipfs/pin"
57 repo "github.com/ipfs/go-ipfs/repo"
@@ -104,6 +105,7 @@ type IpfsNode struct {
105 Diagnostics *diag.Diagnostics // the diagnostics service
106 Ping *ping.PingService
107 Reprovider *rp.Reprovider // the value reprovider system
108 + IpnsRepub *ipnsrp.Republisher
109
110 IpnsFs *ipnsfs.Filesystem
111
@@ -226,6 +228,29 @@ func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost
228 // setup name system
229 n.Namesys = namesys.NewNameSystem(n.Routing)
230
231 + // setup ipns republishing
232 + n.IpnsRepub = ipnsrp.NewRepublisher(n.Routing, n.Repo.Datastore(), n.Peerstore)
233 + n.IpnsRepub.AddName(n.Identity)
234 +
235 + cfg, err := n.Repo.Config()
236 + if err != nil {
237 + return err
238 + }
239 + if cfg.Ipns.RepublishPeriod != "" {
240 + d, err := time.ParseDuration(cfg.Ipns.RepublishPeriod)
241 + if err != nil {
242 + return fmt.Errorf("failure to parse config setting IPNS.RepublishPeriod: %s", err)
243 + }
244 +
245 + if d < time.Minute || d > (time.Hour*24) {
246 + return fmt.Errorf("config setting IPNS.RepublishPeriod is not between 1min and 1day: %s", d)
247 + }
248 +
249 + n.IpnsRepub.Interval = d
250 + }
251 +
252 + n.Process().Go(n.IpnsRepub.Run)
253 +
254 return nil
255 }
256
namesys/publisher.go
+58 -16
@@ -14,6 +14,7 @@ import (
14 dag "github.com/ipfs/go-ipfs/merkledag"
15 pb "github.com/ipfs/go-ipfs/namesys/pb"
16 ci "github.com/ipfs/go-ipfs/p2p/crypto"
17 + peer "github.com/ipfs/go-ipfs/p2p/peer"
18 path "github.com/ipfs/go-ipfs/path"
19 pin "github.com/ipfs/go-ipfs/pin"
20 routing "github.com/ipfs/go-ipfs/routing"
@@ -30,6 +31,8 @@ var ErrExpiredRecord = errors.New("expired record")
31 // unknown validity type.
32 var ErrUnrecognizedValidity = errors.New("unrecognized validity type")
33
34 +var PublishPutValTimeout = time.Minute
35 +
36 // ipnsPublisher is capable of publishing and resolving names to the IPFS
37 // routing system.
38 type ipnsPublisher struct {
@@ -37,7 +40,7 @@ type ipnsPublisher struct {
40 }
41
42 // NewRoutingPublisher constructs a publisher for the IPFS Routing name system.
40 -func NewRoutingPublisher(route routing.IpfsRouting) Publisher {
43 +func NewRoutingPublisher(route routing.IpfsRouting) *ipnsPublisher {
44 return &ipnsPublisher{routing: route}
45 }
46
@@ -45,16 +48,19 @@ func NewRoutingPublisher(route routing.IpfsRouting) Publisher {
48 // and publishes it out to the routing system
49 func (p *ipnsPublisher) Publish(ctx context.Context, k ci.PrivKey, value path.Path) error {
50 log.Debugf("Publish %s", value)
51 + return p.PublishWithEOL(ctx, k, value, time.Now().Add(time.Hour*24))
52 +}
53
49 - pubkey := k.GetPublic()
50 - pkbytes, err := pubkey.Bytes()
54 +// PublishWithEOL is a temporary stand in for the ipns records implementation
55 +// see here for more details: https://github.com/ipfs/specs/tree/master/records
56 +func (p *ipnsPublisher) PublishWithEOL(ctx context.Context, k ci.PrivKey, value path.Path, eol time.Time) error {
57 +
58 + id, err := peer.IDFromPrivateKey(k)
59 if err != nil {
60 return err
61 }
62
55 - nameb := u.Hash(pkbytes)
56 - namekey := key.Key("/pk/" + string(nameb))
57 - ipnskey := key.Key("/ipns/" + string(nameb))
63 + namekey, ipnskey := IpnsKeysForID(id)
64
65 // get previous records sequence number, and add one to it
66 var seqnum uint64
@@ -71,46 +77,75 @@ func (p *ipnsPublisher) Publish(ctx context.Context, k ci.PrivKey, value path.Pa
77 return err
78 }
79
74 - data, err := createRoutingEntryData(k, value, seqnum)
80 + entry, err := CreateRoutingEntryData(k, value, seqnum, eol)
81 + if err != nil {
82 + return err
83 + }
84 +
85 + err = PublishEntry(ctx, p.routing, ipnskey, entry)
86 + if err != nil {
87 + return err
88 + }
89 +
90 + err = PublishPublicKey(ctx, p.routing, namekey, k.GetPublic())
91 + if err != nil {
92 + return err
93 + }
94 +
95 + return nil
96 +}
97 +
98 +func PublishPublicKey(ctx context.Context, r routing.IpfsRouting, k key.Key, pubk ci.PubKey) error {
99 + log.Debugf("Storing pubkey at: %s", k)
100 + pkbytes, err := pubk.Bytes()
101 if err != nil {
102 return err
103 }
104
79 - log.Debugf("Storing pubkey at: %s", namekey)
105 // Store associated public key
81 - timectx, cancel := context.WithDeadline(ctx, time.Now().Add(time.Second*10))
106 + timectx, cancel := context.WithTimeout(ctx, PublishPutValTimeout)
107 defer cancel()
83 - err = p.routing.PutValue(timectx, namekey, pkbytes)
108 + err = r.PutValue(timectx, k, pkbytes)
109 + if err != nil {
110 + return err
111 + }
112 +
113 + return nil
114 +}
115 +
116 +func PublishEntry(ctx context.Context, r routing.IpfsRouting, ipnskey key.Key, rec *pb.IpnsEntry) error {
117 + timectx, cancel := context.WithTimeout(ctx, PublishPutValTimeout)
118 + defer cancel()
119 +
120 + data, err := proto.Marshal(rec)
121 if err != nil {
122 return err
123 }
124
125 log.Debugf("Storing ipns entry at: %s", ipnskey)
126 // Store ipns entry at "/ipns/"+b58(h(pubkey))
90 - timectx, cancel = context.WithDeadline(ctx, time.Now().Add(time.Second*10))
91 - defer cancel()
92 - if err := p.routing.PutValue(timectx, ipnskey, data); err != nil {
127 + if err := r.PutValue(timectx, ipnskey, data); err != nil {
128 return err
129 }
130
131 return nil
132 }
133
99 -func createRoutingEntryData(pk ci.PrivKey, val path.Path, seq uint64) ([]byte, error) {
134 +func CreateRoutingEntryData(pk ci.PrivKey, val path.Path, seq uint64, eol time.Time) (*pb.IpnsEntry, error) {
135 entry := new(pb.IpnsEntry)
136
137 entry.Value = []byte(val)
138 typ := pb.IpnsEntry_EOL
139 entry.ValidityType = &typ
140 entry.Sequence = proto.Uint64(seq)
106 - entry.Validity = []byte(u.FormatRFC3339(time.Now().Add(time.Hour * 24)))
141 + entry.Validity = []byte(u.FormatRFC3339(eol))
142
143 sig, err := pk.Sign(ipnsEntryDataForSig(entry))
144 if err != nil {
145 return nil, err
146 }
147 entry.Signature = sig
113 - return proto.Marshal(entry)
148 + return entry, nil
149 }
150
151 func ipnsEntryDataForSig(e *pb.IpnsEntry) []byte {
@@ -226,3 +261,10 @@ func InitializeKeyspace(ctx context.Context, ds dag.DAGService, pub Publisher, p
261
262 return nil
263 }
264 +
265 +func IpnsKeysForID(id peer.ID) (name, ipns key.Key) {
266 + namekey := key.Key("/pk/" + id)
267 + ipnskey := key.Key("/ipns/" + id)
268 +
269 + return namekey, ipnskey
270 +}
namesys/republisher/repub.go new
+142
@@ -0,0 +1,142 @@
1 +package republisher
2 +
3 +import (
4 + "errors"
5 + "sync"
6 + "time"
7 +
8 + key "github.com/ipfs/go-ipfs/blocks/key"
9 + namesys "github.com/ipfs/go-ipfs/namesys"
10 + pb "github.com/ipfs/go-ipfs/namesys/pb"
11 + peer "github.com/ipfs/go-ipfs/p2p/peer"
12 + path "github.com/ipfs/go-ipfs/path"
13 + "github.com/ipfs/go-ipfs/routing"
14 + dhtpb "github.com/ipfs/go-ipfs/routing/dht/pb"
15 +
16 + proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto"
17 + ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
18 + goprocess "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
19 + gpctx "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess/context"
20 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
21 + logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0"
22 +)
23 +
24 +var errNoEntry = errors.New("no previous entry")
25 +
26 +var log = logging.Logger("ipns-repub")
27 +
28 +var DefaultRebroadcastInterval = time.Hour * 4
29 +
30 +const DefaultRecordLifetime = time.Hour * 24
31 +
32 +type Republisher struct {
33 + r routing.IpfsRouting
34 + ds ds.Datastore
35 + ps peer.Peerstore
36 +
37 + Interval time.Duration
38 +
39 + // how long records that are republished should be valid for
40 + RecordLifetime time.Duration
41 +
42 + entrylock sync.Mutex
43 + entries map[peer.ID]struct{}
44 +}
45 +
46 +func NewRepublisher(r routing.IpfsRouting, ds ds.Datastore, ps peer.Peerstore) *Republisher {
47 + return &Republisher{
48 + r: r,
49 + ps: ps,
50 + ds: ds,
51 + entries: make(map[peer.ID]struct{}),
52 + Interval: DefaultRebroadcastInterval,
53 + RecordLifetime: DefaultRecordLifetime,
54 + }
55 +}
56 +
57 +func (rp *Republisher) AddName(id peer.ID) {
58 + rp.entrylock.Lock()
59 + defer rp.entrylock.Unlock()
60 + rp.entries[id] = struct{}{}
61 +}
62 +
63 +func (rp *Republisher) Run(proc goprocess.Process) {
64 + tick := time.NewTicker(rp.Interval)
65 + defer tick.Stop()
66 +
67 + for {
68 + select {
69 + case <-tick.C:
70 + err := rp.republishEntries(proc)
71 + if err != nil {
72 + log.Error(err)
73 + }
74 + case <-proc.Closing():
75 + return
76 + }
77 + }
78 +}
79 +
80 +func (rp *Republisher) republishEntries(p goprocess.Process) error {
81 + ctx, cancel := context.WithCancel(gpctx.OnClosingContext(p))
82 + defer cancel()
83 +
84 + for id, _ := range rp.entries {
85 + log.Debugf("republishing ipns entry for %s", id)
86 + priv := rp.ps.PrivKey(id)
87 +
88 + // Look for it locally only
89 + namekey, ipnskey := namesys.IpnsKeysForID(id)
90 + p, seq, err := rp.getLastVal(ipnskey)
91 + if err != nil {
92 + if err == errNoEntry {
93 + continue
94 + }
95 + return err
96 + }
97 +
98 + // update record with same sequence number
99 + eol := time.Now().Add(rp.RecordLifetime)
100 + entry, err := namesys.CreateRoutingEntryData(priv, p, seq, eol)
101 + if err != nil {
102 + return err
103 + }
104 +
105 + // republish public key
106 + err = namesys.PublishPublicKey(ctx, rp.r, namekey, priv.GetPublic())
107 + if err != nil {
108 + return err
109 + }
110 +
111 + // republish ipns entry
112 + err = namesys.PublishEntry(ctx, rp.r, ipnskey, entry)
113 + if err != nil {
114 + return err
115 + }
116 + }
117 +
118 + return nil
119 +}
120 +
121 +func (rp *Republisher) getLastVal(k key.Key) (path.Path, uint64, error) {
122 + ival, err := rp.ds.Get(k.DsKey())
123 + if err != nil {
124 + // not found means we dont have a previously published entry
125 + return "", 0, errNoEntry
126 + }
127 +
128 + val := ival.([]byte)
129 + dhtrec := new(dhtpb.Record)
130 + err = proto.Unmarshal(val, dhtrec)
131 + if err != nil {
132 + return "", 0, err
133 + }
134 +
135 + // extract published data from record
136 + e := new(pb.IpnsEntry)
137 + err = proto.Unmarshal(dhtrec.GetValue(), e)
138 + if err != nil {
139 + return "", 0, err
140 + }
141 + return path.Path(e.Value), e.GetSequence(), nil
142 +}
namesys/republisher/repub_test.go new
+120
@@ -0,0 +1,120 @@
1 +package republisher_test
2 +
3 +import (
4 + "errors"
5 + "testing"
6 + "time"
7 +
8 + goprocess "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
9 + context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
10 +
11 + "github.com/ipfs/go-ipfs/core"
12 + mock "github.com/ipfs/go-ipfs/core/mock"
13 + namesys "github.com/ipfs/go-ipfs/namesys"
14 + . "github.com/ipfs/go-ipfs/namesys/republisher"
15 + mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
16 + peer "github.com/ipfs/go-ipfs/p2p/peer"
17 + path "github.com/ipfs/go-ipfs/path"
18 +)
19 +
20 +func TestRepublish(t *testing.T) {
21 + ctx, cancel := context.WithCancel(context.Background())
22 + defer cancel()
23 +
24 + // create network
25 + mn := mocknet.New(ctx)
26 +
27 + var nodes []*core.IpfsNode
28 + for i := 0; i < 10; i++ {
29 + nd, err := core.NewNode(ctx, &core.BuildCfg{
30 + Online: true,
31 + Host: mock.MockHostOption(mn),
32 + })
33 + if err != nil {
34 + t.Fatal(err)
35 + }
36 +
37 + nodes = append(nodes, nd)
38 + }
39 +
40 + mn.LinkAll()
41 +
42 + bsinf := core.BootstrapConfigWithPeers(
43 + []peer.PeerInfo{
44 + nodes[0].Peerstore.PeerInfo(nodes[0].Identity),
45 + },
46 + )
47 +
48 + for _, n := range nodes[1:] {
49 + if err := n.Bootstrap(bsinf); err != nil {
50 + t.Fatal(err)
51 + }
52 + }
53 +
54 + // have one node publish a record that is valid for 1 second
55 + publisher := nodes[3]
56 + p := path.FromString("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid
57 + rp := namesys.NewRoutingPublisher(publisher.Routing)
58 + err := rp.PublishWithEOL(ctx, publisher.PrivateKey, p, time.Now().Add(time.Second))
59 + if err != nil {
60 + t.Fatal(err)
61 + }
62 +
63 + name := "/ipns/" + publisher.Identity.Pretty()
64 + if err := verifyResolution(nodes, name, p); err != nil {
65 + t.Fatal(err)
66 + }
67 +
68 + // Now wait a second, the records will be invalid and we should fail to resolve
69 + time.Sleep(time.Second)
70 + if err := verifyResolutionFails(nodes, name); err != nil {
71 + t.Fatal(err)
72 + }
73 +
74 + // The republishers that are contained within the nodes have their timeout set
75 + // to 12 hours. Instead of trying to tweak those, we're just going to pretend
76 + // they dont exist and make our own.
77 + repub := NewRepublisher(publisher.Routing, publisher.Repo.Datastore(), publisher.Peerstore)
78 + repub.Interval = time.Second
79 + repub.RecordLifetime = time.Second * 5
80 + repub.AddName(publisher.Identity)
81 +
82 + proc := goprocess.Go(repub.Run)
83 + defer proc.Close()
84 +
85 + // now wait a couple seconds for it to fire
86 + time.Sleep(time.Second * 2)
87 +
88 + // we should be able to resolve them now
89 + if err := verifyResolution(nodes, name, p); err != nil {
90 + t.Fatal(err)
91 + }
92 +}
93 +
94 +func verifyResolution(nodes []*core.IpfsNode, key string, exp path.Path) error {
95 + ctx, cancel := context.WithCancel(context.Background())
96 + defer cancel()
97 + for _, n := range nodes {
98 + val, err := n.Namesys.Resolve(ctx, key)
99 + if err != nil {
100 + return err
101 + }
102 +
103 + if val != exp {
104 + return errors.New("resolved wrong record")
105 + }
106 + }
107 + return nil
108 +}
109 +
110 +func verifyResolutionFails(nodes []*core.IpfsNode, key string) error {
111 + ctx, cancel := context.WithCancel(context.Background())
112 + defer cancel()
113 + for _, n := range nodes {
114 + _, err := n.Namesys.Resolve(ctx, key)
115 + if err == nil {
116 + return errors.New("expected resolution to fail")
117 + }
118 + }
119 + return nil
120 +}
repo/config/config.go
+1
@@ -23,6 +23,7 @@ type Config struct {
23 Mounts Mounts // local node's mount points
24 Version Version // local node's version management
25 Discovery Discovery // local node's discovery mechanisms
26 + Ipns Ipns // Ipns settings
27 Bootstrap []string // local nodes's bootstrap peer addresses
28 Tour Tour // local node's tour position
29 Gateway Gateway // local node's gateway server options
repo/config/ipns.go new
+5
@@ -0,0 +1,5 @@
1 +package config
2 +
3 +type Ipns struct {
4 + RepublishPeriod string
5 +}
routing/dht/dht.go
+4 -8
@@ -18,7 +18,6 @@ import (
18 pb "github.com/ipfs/go-ipfs/routing/dht/pb"
19 kb "github.com/ipfs/go-ipfs/routing/kbucket"
20 record "github.com/ipfs/go-ipfs/routing/record"
21 - u "github.com/ipfs/go-ipfs/util"
21 logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0"
22
23 proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto"
@@ -217,13 +216,10 @@ func (dht *IpfsDHT) getLocal(key key.Key) (*pb.Record, error) {
216 return nil, err
217 }
218
220 - // TODO: 'if paranoid'
221 - if u.Debug {
222 - err = dht.verifyRecordLocally(rec)
223 - if err != nil {
224 - log.Debugf("local record verify failed: %s (discarded)", err)
225 - return nil, err
226 - }
219 + err = dht.verifyRecordLocally(rec)
220 + if err != nil {
221 + log.Debugf("local record verify failed: %s (discarded)", err)
222 + return nil, err
223 }
224
225 return rec, nil
test/integration/addcat_test.go
+1 -1
@@ -18,9 +18,9 @@ import (
18 mock "github.com/ipfs/go-ipfs/core/mock"
19 mocknet "github.com/ipfs/go-ipfs/p2p/net/mock"
20 "github.com/ipfs/go-ipfs/p2p/peer"
21 - logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0"
21 "github.com/ipfs/go-ipfs/thirdparty/unit"
22 testutil "github.com/ipfs/go-ipfs/util/testutil"
23 + logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0"
24 )
25
26 var log = logging.Logger("epictest")