@cryptotaxi247 / kubo / commits / b34d41e01

fix publish fail on prexisting bad record

dont error out if prexisting record is bad, just grab its sequence number and continue on with the publish. License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>

Jeromy committed Oct 1, 2015 at 15:02 UTC b34d41e01ed32a1cfbdcd96f2a34c8b38345a56d
12 files changed +275 -61
core/core.go
+2 -2
@@ -227,7 +227,7 @@ func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost
227 n.Exchange = bitswap.New(ctx, n.Identity, bitswapNetwork, n.Blockstore, alwaysSendToPeer)
228
229 // setup name system
230 - n.Namesys = namesys.NewNameSystem(n.Routing)
230 + n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore())
231
232 // setup ipns republishing
233 err = n.setupIpnsRepublisher()
@@ -456,7 +456,7 @@ func (n *IpfsNode) SetupOfflineRouting() error {
456
457 n.Routing = offroute.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
458
459 - n.Namesys = namesys.NewNameSystem(n.Routing)
459 + n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore())
460
461 return nil
462 }
fuse/ipns/common.go
+1 -1
@@ -33,7 +33,7 @@ func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
33 return err
34 }
35
36 - pub := nsys.NewRoutingPublisher(n.Routing)
36 + pub := nsys.NewRoutingPublisher(n.Routing, n.Repo.Datastore())
37 if err := pub.Publish(ctx, key, path.FromKey(nodek)); err != nil {
38 return err
39 }
namesys/namesys.go
+3 -2
@@ -4,6 +4,7 @@ import (
4 "strings"
5 "time"
6
7 + ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8 context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
9 ci "github.com/ipfs/go-ipfs/p2p/crypto"
10 path "github.com/ipfs/go-ipfs/path"
@@ -25,7 +26,7 @@ type mpns struct {
26 }
27
28 // NewNameSystem will construct the IPFS naming system based on Routing
28 -func NewNameSystem(r routing.IpfsRouting) NameSystem {
29 +func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore) NameSystem {
30 return &mpns{
31 resolvers: map[string]resolver{
32 "dns": newDNSResolver(),
@@ -33,7 +34,7 @@ func NewNameSystem(r routing.IpfsRouting) NameSystem {
34 "dht": newRoutingResolver(r),
35 },
36 publishers: map[string]Publisher{
36 - "/ipns/": NewRoutingPublisher(r),
37 + "/ipns/": NewRoutingPublisher(r, ds),
38 },
39 }
40 }
namesys/publisher.go
+53 -12
@@ -18,6 +18,7 @@ import (
18 path "github.com/ipfs/go-ipfs/path"
19 pin "github.com/ipfs/go-ipfs/pin"
20 routing "github.com/ipfs/go-ipfs/routing"
21 + dhtpb "github.com/ipfs/go-ipfs/routing/dht/pb"
22 record "github.com/ipfs/go-ipfs/routing/record"
23 ft "github.com/ipfs/go-ipfs/unixfs"
24 u "github.com/ipfs/go-ipfs/util"
@@ -37,11 +38,15 @@ var PublishPutValTimeout = time.Minute
38 // routing system.
39 type ipnsPublisher struct {
40 routing routing.IpfsRouting
41 + ds ds.Datastore
42 }
43
44 // NewRoutingPublisher constructs a publisher for the IPFS Routing name system.
43 -func NewRoutingPublisher(route routing.IpfsRouting) *ipnsPublisher {
44 - return &ipnsPublisher{routing: route}
45 +func NewRoutingPublisher(route routing.IpfsRouting, ds ds.Datastore) *ipnsPublisher {
46 + if ds == nil {
47 + panic("nil datastore")
48 + }
49 + return &ipnsPublisher{routing: route, ds: ds}
50 }
51
52 // Publish implements Publisher. Accepts a keypair and a value,
@@ -62,22 +67,58 @@ func (p *ipnsPublisher) PublishWithEOL(ctx context.Context, k ci.PrivKey, value
67
68 _, ipnskey := IpnsKeysForID(id)
69
65 - // get previous records sequence number, and add one to it
66 - var seqnum uint64
67 - prevrec, err := p.routing.GetValues(ctx, ipnskey, 0)
70 + // get previous records sequence number
71 + seqnum, err := p.getPreviousSeqNo(ctx, ipnskey)
72 + if err != nil {
73 + return err
74 + }
75 +
76 + // increment it
77 + seqnum++
78 +
79 + return PutRecordToRouting(ctx, k, value, seqnum, eol, p.routing, id)
80 +}
81 +
82 +func (p *ipnsPublisher) getPreviousSeqNo(ctx context.Context, ipnskey key.Key) (uint64, error) {
83 + prevrec, err := p.ds.Get(ipnskey.DsKey())
84 + if err != nil && err != ds.ErrNotFound {
85 + // None found, lets start at zero!
86 + return 0, err
87 + }
88 + var val []byte
89 if err == nil {
69 - e := new(pb.IpnsEntry)
70 - err := proto.Unmarshal(prevrec[0].Val, e)
90 + prbytes, ok := prevrec.([]byte)
91 + if !ok {
92 + return 0, fmt.Errorf("unexpected type returned from datastore: %#v", prevrec)
93 + }
94 + dhtrec := new(dhtpb.Record)
95 + err := proto.Unmarshal(prbytes, dhtrec)
96 if err != nil {
72 - return err
97 + return 0, err
98 }
99
75 - seqnum = e.GetSequence() + 1
76 - } else if err != ds.ErrNotFound {
77 - return err
100 + val = dhtrec.GetValue()
101 + } else {
102 + // try and check the dht for a record
103 + ctx, cancel := context.WithTimeout(ctx, time.Second*30)
104 + defer cancel()
105 +
106 + rv, err := p.routing.GetValue(ctx, ipnskey)
107 + if err != nil {
108 + // no such record found, start at zero!
109 + return 0, nil
110 + }
111 +
112 + val = rv
113 }
114
80 - return PutRecordToRouting(ctx, k, value, seqnum, eol, p.routing, id)
115 + e := new(pb.IpnsEntry)
116 + err = proto.Unmarshal(val, e)
117 + if err != nil {
118 + return 0, err
119 + }
120 +
121 + return e.GetSequence(), nil
122 }
123
124 func PutRecordToRouting(ctx context.Context, k ci.PrivKey, value path.Path, seqnum uint64, eol time.Time, r routing.IpfsRouting, id peer.ID) error {
namesys/republisher/repub_test.go
+1 -1
@@ -54,7 +54,7 @@ func TestRepublish(t *testing.T) {
54 // have one node publish a record that is valid for 1 second
55 publisher := nodes[3]
56 p := path.FromString("/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn") // does not need to be valid
57 - rp := namesys.NewRoutingPublisher(publisher.Routing)
57 + rp := namesys.NewRoutingPublisher(publisher.Routing, publisher.Repo.Datastore())
58 err := rp.PublishWithEOL(ctx, publisher.PrivateKey, p, time.Now().Add(time.Second))
59 if err != nil {
60 t.Fatal(err)
namesys/resolve_test.go
+93 -1
@@ -1,10 +1,14 @@
1 package namesys
2
3 import (
4 + "errors"
5 "testing"
6 + "time"
7
8 + ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
9 context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
10 key "github.com/ipfs/go-ipfs/blocks/key"
11 + peer "github.com/ipfs/go-ipfs/p2p/peer"
12 path "github.com/ipfs/go-ipfs/path"
13 mockrouting "github.com/ipfs/go-ipfs/routing/mock"
14 u "github.com/ipfs/go-ipfs/util"
@@ -13,9 +17,10 @@ import (
17
18 func TestRoutingResolve(t *testing.T) {
19 d := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t))
20 + dstore := ds.NewMapDatastore()
21
22 resolver := NewRoutingResolver(d)
18 - publisher := NewRoutingPublisher(d)
23 + publisher := NewRoutingPublisher(d, dstore)
24
25 privk, pubk, err := testutil.RandTestKeyPair(512)
26 if err != nil {
@@ -43,3 +48,90 @@ func TestRoutingResolve(t *testing.T) {
48 t.Fatal("Got back incorrect value.")
49 }
50 }
51 +
52 +func TestPrexistingExpiredRecord(t *testing.T) {
53 + dstore := ds.NewMapDatastore()
54 + d := mockrouting.NewServer().ClientWithDatastore(context.Background(), testutil.RandIdentityOrFatal(t), dstore)
55 +
56 + resolver := NewRoutingResolver(d)
57 + publisher := NewRoutingPublisher(d, dstore)
58 +
59 + privk, pubk, err := testutil.RandTestKeyPair(512)
60 + if err != nil {
61 + t.Fatal(err)
62 + }
63 +
64 + id, err := peer.IDFromPublicKey(pubk)
65 + if err != nil {
66 + t.Fatal(err)
67 + }
68 +
69 + // Make an expired record and put it in the datastore
70 + h := path.FromString("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN")
71 + eol := time.Now().Add(time.Hour * -1)
72 + err = PutRecordToRouting(context.Background(), privk, h, 0, eol, d, id)
73 + if err != nil {
74 + t.Fatal(err)
75 + }
76 +
77 + // Now, with an old record in the system already, try and publish a new one
78 + err = publisher.Publish(context.Background(), privk, h)
79 + if err != nil {
80 + t.Fatal(err)
81 + }
82 +
83 + err = verifyCanResolve(resolver, id.Pretty(), h)
84 + if err != nil {
85 + t.Fatal(err)
86 + }
87 +}
88 +
89 +func TestPrexistingRecord(t *testing.T) {
90 + dstore := ds.NewMapDatastore()
91 + d := mockrouting.NewServer().ClientWithDatastore(context.Background(), testutil.RandIdentityOrFatal(t), dstore)
92 +
93 + resolver := NewRoutingResolver(d)
94 + publisher := NewRoutingPublisher(d, dstore)
95 +
96 + privk, pubk, err := testutil.RandTestKeyPair(512)
97 + if err != nil {
98 + t.Fatal(err)
99 + }
100 +
101 + id, err := peer.IDFromPublicKey(pubk)
102 + if err != nil {
103 + t.Fatal(err)
104 + }
105 +
106 + // Make a good record and put it in the datastore
107 + h := path.FromString("/ipfs/QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN")
108 + eol := time.Now().Add(time.Hour)
109 + err = PutRecordToRouting(context.Background(), privk, h, 0, eol, d, id)
110 + if err != nil {
111 + t.Fatal(err)
112 + }
113 +
114 + // Now, with an old record in the system already, try and publish a new one
115 + err = publisher.Publish(context.Background(), privk, h)
116 + if err != nil {
117 + t.Fatal(err)
118 + }
119 +
120 + err = verifyCanResolve(resolver, id.Pretty(), h)
121 + if err != nil {
122 + t.Fatal(err)
123 + }
124 +}
125 +
126 +func verifyCanResolve(r Resolver, name string, exp path.Path) error {
127 + res, err := r.Resolve(context.Background(), name)
128 + if err != nil {
129 + return err
130 + }
131 +
132 + if res != exp {
133 + return errors.New("got back wrong record!")
134 + }
135 +
136 + return nil
137 +}
routing/dht/handlers.go
+75 -29
@@ -3,6 +3,7 @@ package dht
3 import (
4 "errors"
5 "fmt"
6 + "time"
7
8 proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto"
9 ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
@@ -10,6 +11,7 @@ import (
11 key "github.com/ipfs/go-ipfs/blocks/key"
12 peer "github.com/ipfs/go-ipfs/p2p/peer"
13 pb "github.com/ipfs/go-ipfs/routing/dht/pb"
14 + u "github.com/ipfs/go-ipfs/util"
15 lgbl "github.com/ipfs/go-ipfs/util/eventlog/loggables"
16 )
17
@@ -46,41 +48,17 @@ func (dht *IpfsDHT) handleGetValue(ctx context.Context, p peer.ID, pmes *pb.Mess
48 resp := pb.NewMessage(pmes.GetType(), pmes.GetKey(), pmes.GetClusterLevel())
49
50 // first, is there even a key?
49 - k := pmes.GetKey()
51 + k := key.Key(pmes.GetKey())
52 if k == "" {
53 return nil, errors.New("handleGetValue but no key was provided")
54 // TODO: send back an error response? could be bad, but the other node's hanging.
55 }
56
55 - // let's first check if we have the value locally.
56 - log.Debugf("%s handleGetValue looking into ds", dht.self)
57 - dskey := key.Key(k).DsKey()
58 - iVal, err := dht.datastore.Get(dskey)
59 - log.Debugf("%s handleGetValue looking into ds GOT %v", dht.self, iVal)
60 -
61 - // if we got an unexpected error, bail.
62 - if err != nil && err != ds.ErrNotFound {
57 + rec, err := dht.checkLocalDatastore(k)
58 + if err != nil {
59 return nil, err
60 }
65 -
66 - // if we have the value, send it back
67 - if err == nil {
68 - log.Debugf("%s handleGetValue success!", dht.self)
69 -
70 - byts, ok := iVal.([]byte)
71 - if !ok {
72 - return nil, fmt.Errorf("datastore had non byte-slice value for %v", dskey)
73 - }
74 -
75 - rec := new(pb.Record)
76 - err := proto.Unmarshal(byts, rec)
77 - if err != nil {
78 - log.Debug("Failed to unmarshal dht record from datastore")
79 - return nil, err
80 - }
81 -
82 - resp.Record = rec
83 - }
61 + resp.Record = rec
62
63 // Find closest peer on given cluster to desired key and reply with that info
64 closer := dht.betterPeersToQuery(pmes, p, CloserPeerCount)
@@ -102,6 +80,69 @@ func (dht *IpfsDHT) handleGetValue(ctx context.Context, p peer.ID, pmes *pb.Mess
80 return resp, nil
81 }
82
83 +func (dht *IpfsDHT) checkLocalDatastore(k key.Key) (*pb.Record, error) {
84 + log.Debugf("%s handleGetValue looking into ds", dht.self)
85 + dskey := k.DsKey()
86 + iVal, err := dht.datastore.Get(dskey)
87 + log.Debugf("%s handleGetValue looking into ds GOT %v", dht.self, iVal)
88 +
89 + if err == ds.ErrNotFound {
90 + return nil, nil
91 + }
92 +
93 + // if we got an unexpected error, bail.
94 + if err != nil {
95 + return nil, err
96 + }
97 +
98 + // if we have the value, send it back
99 + log.Debugf("%s handleGetValue success!", dht.self)
100 +
101 + byts, ok := iVal.([]byte)
102 + if !ok {
103 + return nil, fmt.Errorf("datastore had non byte-slice value for %v", dskey)
104 + }
105 +
106 + rec := new(pb.Record)
107 + err = proto.Unmarshal(byts, rec)
108 + if err != nil {
109 + log.Debug("Failed to unmarshal dht record from datastore")
110 + return nil, err
111 + }
112 +
113 + // if its our record, dont bother checking the times on it
114 + if peer.ID(rec.GetAuthor()) == dht.self {
115 + return rec, nil
116 + }
117 +
118 + var recordIsBad bool
119 + recvtime, err := u.ParseRFC3339(rec.GetTimeReceived())
120 + if err != nil {
121 + log.Info("either no receive time set on record, or it was invalid: ", err)
122 + recordIsBad = true
123 + }
124 +
125 + if time.Now().Sub(recvtime) > MaxRecordAge {
126 + log.Debug("old record found, tossing.")
127 + recordIsBad = true
128 + }
129 +
130 + // NOTE: we do not verify the record here beyond checking these timestamps.
131 + // we put the burden of checking the records on the requester as checking a record
132 + // may be computationally expensive
133 +
134 + if recordIsBad {
135 + err := dht.datastore.Delete(dskey)
136 + if err != nil {
137 + log.Error("Failed to delete bad record from datastore: ", err)
138 + }
139 +
140 + return nil, nil // can treat this as not having the record at all
141 + }
142 +
143 + return rec, nil
144 +}
145 +
146 // Store a value in this peer local storage
147 func (dht *IpfsDHT) handlePutValue(ctx context.Context, p peer.ID, pmes *pb.Message) (*pb.Message, error) {
148 defer log.EventBegin(ctx, "handlePutValue", p).Done()
@@ -112,7 +153,12 @@ func (dht *IpfsDHT) handlePutValue(ctx context.Context, p peer.ID, pmes *pb.Mess
153 return nil, err
154 }
155
115 - data, err := proto.Marshal(pmes.GetRecord())
156 + rec := pmes.GetRecord()
157 +
158 + // record the time we receive every record
159 + rec.TimeReceived = proto.String(u.FormatRFC3339(time.Now()))
160 +
161 + data, err := proto.Marshal(rec)
162 if err != nil {
163 return nil, err
164 }
routing/dht/pb/dht.pb.go
+12 -3
@@ -14,7 +14,7 @@ It has these top-level messages:
14 */
15 package dht_pb
16
17 -import proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto"
17 +import proto "github.com/gogo/protobuf/proto"
18 import math "math"
19
20 // Reference imports to suppress errors if they are not otherwise used.
@@ -221,8 +221,10 @@ type Record struct {
221 // hash of the authors public key
222 Author *string `protobuf:"bytes,3,opt,name=author" json:"author,omitempty"`
223 // A PKI signature for the key+value+author
224 - Signature []byte `protobuf:"bytes,4,opt,name=signature" json:"signature,omitempty"`
225 - XXX_unrecognized []byte `json:"-"`
224 + Signature []byte `protobuf:"bytes,4,opt,name=signature" json:"signature,omitempty"`
225 + // Time the record was received, set by receiver
226 + TimeReceived *string `protobuf:"bytes,5,opt,name=timeReceived" json:"timeReceived,omitempty"`
227 + XXX_unrecognized []byte `json:"-"`
228 }
229
230 func (m *Record) Reset() { *m = Record{} }
@@ -257,6 +259,13 @@ func (m *Record) GetSignature() []byte {
259 return nil
260 }
261
262 +func (m *Record) GetTimeReceived() string {
263 + if m != nil && m.TimeReceived != nil {
264 + return *m.TimeReceived
265 + }
266 + return ""
267 +}
268 +
269 func init() {
270 proto.RegisterEnum("dht.pb.Message_MessageType", Message_MessageType_name, Message_MessageType_value)
271 proto.RegisterEnum("dht.pb.Message_ConnectionType", Message_ConnectionType_name, Message_ConnectionType_value)
routing/dht/pb/dht.proto
+3
@@ -75,4 +75,7 @@ message Record {
75
76 // A PKI signature for the key+value+author
77 optional bytes signature = 4;
78 +
79 + // Time the record was received, set by receiver
80 + optional string timeReceived = 5;
81 }
routing/dht/records.go
+9
@@ -2,6 +2,7 @@ package dht
2
3 import (
4 "fmt"
5 + "time"
6
7 ctxfrac "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-context/frac"
8 "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
@@ -12,6 +13,14 @@ import (
13 record "github.com/ipfs/go-ipfs/routing/record"
14 )
15
16 +// MaxRecordAge specifies the maximum time that any node will hold onto a record
17 +// from the time its received. This does not apply to any other forms of validity that
18 +// the record may contain.
19 +// For example, a record may contain an ipns entry with an EOL saying its valid
20 +// until the year 2020 (a great time in the future). For that record to stick around
21 +// it must be rebroadcasted more frequently than once every 'MaxRecordAge'
22 +const MaxRecordAge = time.Hour * 36
23 +
24 func (dht *IpfsDHT) GetPublicKey(ctx context.Context, p peer.ID) (ci.PubKey, error) {
25 log.Debugf("getPublicKey for: %s", p)
26
routing/mock/centralized_client.go
+22 -9
@@ -4,12 +4,15 @@ import (
4 "errors"
5 "time"
6
7 + proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto"
8 ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
9 ma "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multiaddr"
10 context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
11 key "github.com/ipfs/go-ipfs/blocks/key"
12 peer "github.com/ipfs/go-ipfs/p2p/peer"
13 routing "github.com/ipfs/go-ipfs/routing"
14 + dhtpb "github.com/ipfs/go-ipfs/routing/dht/pb"
15 + u "github.com/ipfs/go-ipfs/util"
16 "github.com/ipfs/go-ipfs/util/testutil"
17 logging "github.com/ipfs/go-ipfs/vendor/go-log-v1.0.0"
18 )
@@ -25,7 +28,16 @@ type client struct {
28 // FIXME(brian): is this method meant to simulate putting a value into the network?
29 func (c *client) PutValue(ctx context.Context, key key.Key, val []byte) error {
30 log.Debugf("PutValue: %s", key)
28 - return c.datastore.Put(key.DsKey(), val)
31 + rec := new(dhtpb.Record)
32 + rec.Value = val
33 + rec.Key = proto.String(string(key))
34 + rec.TimeReceived = proto.String(u.FormatRFC3339(time.Now()))
35 + data, err := proto.Marshal(rec)
36 + if err != nil {
37 + return err
38 + }
39 +
40 + return c.datastore.Put(key.DsKey(), data)
41 }
42
43 // FIXME(brian): is this method meant to simulate getting a value from the network?
@@ -41,21 +53,22 @@ func (c *client) GetValue(ctx context.Context, key key.Key) ([]byte, error) {
53 return nil, errors.New("could not cast value from datastore")
54 }
55
44 - return data, nil
56 + rec := new(dhtpb.Record)
57 + err = proto.Unmarshal(data, rec)
58 + if err != nil {
59 + return nil, err
60 + }
61 +
62 + return rec.GetValue(), nil
63 }
64
65 func (c *client) GetValues(ctx context.Context, key key.Key, count int) ([]routing.RecvdVal, error) {
48 - log.Debugf("GetValue: %s", key)
49 - v, err := c.datastore.Get(key.DsKey())
66 + log.Debugf("GetValues: %s", key)
67 + data, err := c.GetValue(ctx, key)
68 if err != nil {
69 return nil, err
70 }
71
54 - data, ok := v.([]byte)
55 - if !ok {
56 - return nil, errors.New("could not cast value from datastore")
57 - }
58 -
72 return []routing.RecvdVal{{Val: data, From: c.peer.ID()}}, nil
73 }
74
routing/mock/centralized_server.go
+1 -1
@@ -80,7 +80,7 @@ func (rs *s) Client(p testutil.Identity) Client {
80 func (rs *s) ClientWithDatastore(_ context.Context, p testutil.Identity, datastore ds.Datastore) Client {
81 return &client{
82 peer: p,
83 - datastore: ds.NewMapDatastore(),
83 + datastore: datastore,
84 server: rs,
85 }
86 }