cache ipns entries to speed things up a little
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Oct 22, 2015 at 16:27 UTC
c7fb7ce17d20961cae549667fdbda76a2639fe00
14 files changed
+210
-26
core/commands/ipns.go
+23
-5
@@ -45,6 +45,7 @@ Resolve the value of another name:
45
},
46
Options: []cmds.Option{
47
cmds.BoolOption("recursive", "r", "Resolve until the result is not an IPNS name"),
48
+ cmds.BoolOption("nocache", "n", "Do not used cached entries"),
49
},
50
Run: func(req cmds.Request, res cmds.Response) {
51
@@ -62,13 +63,27 @@ Resolve the value of another name:
63
}
64
}
65
65
- router := n.Routing
66
- if local, _, _ := req.Option("local").Bool(); local {
67
- router = offline.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
66
+ nocache, _, _ := req.Option("nocache").Bool()
67
+ local, _, _ := req.Option("local").Bool()
68
+
69
+ // default to nodes namesys resolver
70
+ var resolver namesys.Resolver = n.Namesys
71
+
72
+ if local && nocache {
73
+ res.SetError(errors.New("cannot specify both local and nocache"), cmds.ErrNormal)
74
+ return
75
}
76
70
- var name string
77
+ if local {
78
+ offroute := offline.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
79
+ resolver = namesys.NewRoutingResolver(offroute, 0)
80
+ }
81
82
+ if nocache {
83
+ resolver = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), 0)
84
+ }
85
+
86
+ var name string
87
if len(req.Arguments()) == 0 {
88
if n.Identity == "" {
89
res.SetError(errors.New("Identity not loaded!"), cmds.ErrNormal)
@@ -86,7 +101,10 @@ Resolve the value of another name:
101
depth = namesys.DefaultDepthLimit
102
}
103
89
- resolver := namesys.NewRoutingResolver(router)
104
+ if !strings.HasPrefix(name, "/ipns/") {
105
+ name = "/ipns/" + name
106
+ }
107
+
108
output, err := resolver.ResolveN(req.Context(), name, depth)
109
if err != nil {
110
res.SetError(err, cmds.ErrNormal)
core/commands/publish.go
+13
-1
@@ -52,6 +52,7 @@ Publish an <ipfs-path> to another public key (not implemented):
52
Options: []cmds.Option{
53
cmds.BoolOption("resolve", "resolve given path before publishing (default=true)"),
54
cmds.StringOption("lifetime", "t", "time duration that the record will be valid for (default: 24hrs)"),
55
+ cmds.StringOption("ttl", "time duration this record should be cached for (caution: experimental)"),
56
},
57
Run: func(req cmds.Request, res cmds.Response) {
58
log.Debug("Begin Publish")
@@ -96,7 +97,18 @@ Publish an <ipfs-path> to another public key (not implemented):
97
popts.pubValidTime = d
98
}
99
99
- output, err := publish(req.Context(), n, n.PrivateKey, path.Path(pstr), popts)
100
+ ctx := req.Context()
101
+ if ttl, found, _ := req.Option("ttl").String(); found {
102
+ d, err := time.ParseDuration(ttl)
103
+ if err != nil {
104
+ res.SetError(err, cmds.ErrNormal)
105
+ return
106
+ }
107
+
108
+ ctx = context.WithValue(ctx, "ipns-publish-ttl", d)
109
+ }
110
+
111
+ output, err := publish(ctx, n, n.PrivateKey, path.Path(pstr), popts)
112
if err != nil {
113
res.SetError(err, cmds.ErrNormal)
114
return
core/core.go
+29
-2
@@ -226,8 +226,13 @@ func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost
226
bitswapNetwork := bsnet.NewFromIpfsHost(n.PeerHost, n.Routing)
227
n.Exchange = bitswap.New(ctx, n.Identity, bitswapNetwork, n.Blockstore, alwaysSendToPeer)
228
229
+ size, err := n.getCacheSize()
230
+ if err != nil {
231
+ return err
232
+ }
233
+
234
// setup name system
230
- n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore())
235
+ n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), size)
236
237
// setup ipns republishing
238
err = n.setupIpnsRepublisher()
@@ -238,6 +243,23 @@ func (n *IpfsNode) startOnlineServicesWithHost(ctx context.Context, host p2phost
243
return nil
244
}
245
246
+// getCacheSize returns cache life and cache size
247
+func (n *IpfsNode) getCacheSize() (int, error) {
248
+ cfg, err := n.Repo.Config()
249
+ if err != nil {
250
+ return 0, err
251
+ }
252
+
253
+ cs := cfg.Ipns.ResolveCacheSize
254
+ if cs == 0 {
255
+ cs = 128
256
+ }
257
+ if cs < 0 {
258
+ return 0, fmt.Errorf("cannot specify negative resolve cache size")
259
+ }
260
+ return cs, nil
261
+}
262
+
263
func (n *IpfsNode) setupIpnsRepublisher() error {
264
cfg, err := n.Repo.Config()
265
if err != nil {
@@ -456,7 +478,12 @@ func (n *IpfsNode) SetupOfflineRouting() error {
478
479
n.Routing = offroute.NewOfflineRouter(n.Repo.Datastore(), n.PrivateKey)
480
459
- n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore())
481
+ size, err := n.getCacheSize()
482
+ if err != nil {
483
+ return err
484
+ }
485
+
486
+ n.Namesys = namesys.NewNameSystem(n.Routing, n.Repo.Datastore(), size)
487
488
return nil
489
}
fuse/ipns/ipns_test.go
+1
-1
@@ -113,7 +113,7 @@ func setupIpnsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.M
113
}
114
115
node.Routing = offroute.NewOfflineRouter(node.Repo.Datastore(), node.PrivateKey)
116
- node.Namesys = namesys.NewNameSystem(node.Routing, node.Repo.Datastore())
116
+ node.Namesys = namesys.NewNameSystem(node.Routing, node.Repo.Datastore(), 0)
117
118
ipnsfs, err := nsfs.NewFilesystem(context.Background(), node.DAG, node.Namesys, node.Pinning, node.PrivateKey)
119
if err != nil {
namesys/namesys.go
+4
-2
@@ -26,12 +26,12 @@ type mpns struct {
26
}
27
28
// NewNameSystem will construct the IPFS naming system based on Routing
29
-func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore) NameSystem {
29
+func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore, cachesize int) NameSystem {
30
return &mpns{
31
resolvers: map[string]resolver{
32
"dns": newDNSResolver(),
33
"proquint": new(ProquintResolver),
34
- "dht": newRoutingResolver(r),
34
+ "dht": NewRoutingResolver(r, cachesize),
35
},
36
publishers: map[string]Publisher{
37
"/ipns/": NewRoutingPublisher(r, ds),
@@ -39,6 +39,8 @@ func NewNameSystem(r routing.IpfsRouting, ds ds.Datastore) NameSystem {
39
}
40
}
41
42
+const DefaultResolverCacheTTL = time.Minute
43
+
44
// Resolve implements Resolver.
45
func (ns *mpns) Resolve(ctx context.Context, name string) (path.Path, error) {
46
return ns.ResolveN(ctx, name, DefaultDepthLimit)
namesys/pb/namesys.pb.go
+8
@@ -57,6 +57,7 @@ type IpnsEntry struct {
57
ValidityType *IpnsEntry_ValidityType `protobuf:"varint,3,opt,name=validityType,enum=namesys.pb.IpnsEntry_ValidityType" json:"validityType,omitempty"`
58
Validity []byte `protobuf:"bytes,4,opt,name=validity" json:"validity,omitempty"`
59
Sequence *uint64 `protobuf:"varint,5,opt,name=sequence" json:"sequence,omitempty"`
60
+ Ttl *uint64 `protobuf:"varint,6,opt,name=ttl" json:"ttl,omitempty"`
61
XXX_unrecognized []byte `json:"-"`
62
}
63
@@ -99,6 +100,13 @@ func (m *IpnsEntry) GetSequence() uint64 {
100
return 0
101
}
102
103
+func (m *IpnsEntry) GetTtl() uint64 {
104
+ if m != nil && m.Ttl != nil {
105
+ return *m.Ttl
106
+ }
107
+ return 0
108
+}
109
+
110
func init() {
111
proto.RegisterEnum("namesys.pb.IpnsEntry_ValidityType", IpnsEntry_ValidityType_name, IpnsEntry_ValidityType_value)
112
}
namesys/pb/namesys.proto
+2
@@ -12,4 +12,6 @@ message IpnsEntry {
12
optional bytes validity = 4;
13
14
optional uint64 sequence = 5;
15
+
16
+ optional uint64 ttl = 6;
17
}
namesys/publisher.go
+18
@@ -121,6 +121,19 @@ func (p *ipnsPublisher) getPreviousSeqNo(ctx context.Context, ipnskey key.Key) (
121
return e.GetSequence(), nil
122
}
123
124
+// setting the TTL on published records is an experimental feature.
125
+// as such, i'm using the context to wire it through to avoid changing too
126
+// much code along the way.
127
+func checkCtxTTL(ctx context.Context) (time.Duration, bool) {
128
+ v := ctx.Value("ipns-publish-ttl")
129
+ if v == nil {
130
+ return 0, false
131
+ }
132
+
133
+ d, ok := v.(time.Duration)
134
+ return d, ok
135
+}
136
+
137
func PutRecordToRouting(ctx context.Context, k ci.PrivKey, value path.Path, seqnum uint64, eol time.Time, r routing.IpfsRouting, id peer.ID) error {
138
namekey, ipnskey := IpnsKeysForID(id)
139
entry, err := CreateRoutingEntryData(k, value, seqnum, eol)
@@ -128,6 +141,11 @@ func PutRecordToRouting(ctx context.Context, k ci.PrivKey, value path.Path, seqn
141
return err
142
}
143
144
+ ttl, ok := checkCtxTTL(ctx)
145
+ if ok {
146
+ entry.Ttl = proto.Uint64(uint64(ttl.Nanoseconds()))
147
+ }
148
+
149
err = PublishEntry(ctx, r, ipnskey, entry)
150
if err != nil {
151
return err
namesys/republisher/repub_test.go
+4
@@ -18,6 +18,8 @@ import (
18
)
19
20
func TestRepublish(t *testing.T) {
21
+ // set cache life to zero for testing low-period repubs
22
+
23
ctx, cancel := context.WithCancel(context.Background())
24
defer cancel()
25
@@ -34,6 +36,8 @@ func TestRepublish(t *testing.T) {
36
t.Fatal(err)
37
}
38
39
+ nd.Namesys = namesys.NewNameSystem(nd.Routing, nd.Repo.Datastore(), 0)
40
+
41
nodes = append(nodes, nd)
42
}
43
namesys/resolve_test.go
+3
-3
@@ -19,7 +19,7 @@ func TestRoutingResolve(t *testing.T) {
19
d := mockrouting.NewServer().Client(testutil.RandIdentityOrFatal(t))
20
dstore := ds.NewMapDatastore()
21
22
- resolver := NewRoutingResolver(d)
22
+ resolver := NewRoutingResolver(d, 0)
23
publisher := NewRoutingPublisher(d, dstore)
24
25
privk, pubk, err := testutil.RandTestKeyPair(512)
@@ -53,7 +53,7 @@ func TestPrexistingExpiredRecord(t *testing.T) {
53
dstore := ds.NewMapDatastore()
54
d := mockrouting.NewServer().ClientWithDatastore(context.Background(), testutil.RandIdentityOrFatal(t), dstore)
55
56
- resolver := NewRoutingResolver(d)
56
+ resolver := NewRoutingResolver(d, 0)
57
publisher := NewRoutingPublisher(d, dstore)
58
59
privk, pubk, err := testutil.RandTestKeyPair(512)
@@ -90,7 +90,7 @@ func TestPrexistingRecord(t *testing.T) {
90
dstore := ds.NewMapDatastore()
91
d := mockrouting.NewServer().ClientWithDatastore(context.Background(), testutil.RandIdentityOrFatal(t), dstore)
92
93
- resolver := NewRoutingResolver(d)
93
+ resolver := NewRoutingResolver(d, 0)
94
publisher := NewRoutingPublisher(d, dstore)
95
96
privk, pubk, err := testutil.RandTestKeyPair(512)
namesys/routing.go
+98
-12
@@ -2,16 +2,19 @@ package namesys
2
3
import (
4
"fmt"
5
+ "time"
6
7
proto "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/gogo/protobuf/proto"
8
+ lru "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/hashicorp/golang-lru"
9
mh "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
10
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
11
+ logging "github.com/ipfs/go-ipfs/vendor/QmTBXYb6y2ZcJmoXVKk3pf9rzSEjbCg7tQaJW7RSuH14nv/go-log"
12
13
key "github.com/ipfs/go-ipfs/blocks/key"
14
pb "github.com/ipfs/go-ipfs/namesys/pb"
15
path "github.com/ipfs/go-ipfs/path"
16
routing "github.com/ipfs/go-ipfs/routing"
14
- logging "github.com/ipfs/go-ipfs/vendor/QmTBXYb6y2ZcJmoXVKk3pf9rzSEjbCg7tQaJW7RSuH14nv/go-log"
17
+ u "github.com/ipfs/go-ipfs/util"
18
)
19
20
var log = logging.Logger("namesys")
@@ -19,25 +22,84 @@ var log = logging.Logger("namesys")
22
// routingResolver implements NSResolver for the main IPFS SFS-like naming
23
type routingResolver struct {
24
routing routing.IpfsRouting
25
+
26
+ cache *lru.Cache
27
}
28
24
-// NewRoutingResolver constructs a name resolver using the IPFS Routing system
25
-// to implement SFS-like naming on top.
26
-func NewRoutingResolver(route routing.IpfsRouting) Resolver {
27
- if route == nil {
28
- panic("attempt to create resolver with nil routing system")
29
+func (r *routingResolver) cacheGet(name string) (path.Path, bool) {
30
+ if r.cache == nil {
31
+ return "", false
32
+ }
33
+
34
+ ientry, ok := r.cache.Get(name)
35
+ if !ok {
36
+ return "", false
37
}
38
31
- return &routingResolver{routing: route}
39
+ entry, ok := ientry.(cacheEntry)
40
+ if !ok {
41
+ // should never happen, purely for sanity
42
+ log.Panicf("unexpected type %T in cache for %q.", ientry, name)
43
+ }
44
+
45
+ if time.Now().Before(entry.eol) {
46
+ return entry.val, true
47
+ }
48
+
49
+ r.cache.Remove(name)
50
+
51
+ return "", false
52
}
53
34
-// newRoutingResolver returns a resolver instead of a Resolver.
35
-func newRoutingResolver(route routing.IpfsRouting) resolver {
54
+func (r *routingResolver) cacheSet(name string, val path.Path, rec *pb.IpnsEntry) {
55
+ if r.cache == nil {
56
+ return
57
+ }
58
+
59
+ // if completely unspecified, just use one minute
60
+ ttl := DefaultResolverCacheTTL
61
+ if rec.Ttl != nil {
62
+ recttl := time.Duration(rec.GetTtl())
63
+ if recttl >= 0 {
64
+ ttl = recttl
65
+ }
66
+ }
67
+
68
+ cacheTil := time.Now().Add(ttl)
69
+ eol, ok := checkEOL(rec)
70
+ if ok && eol.Before(cacheTil) {
71
+ cacheTil = eol
72
+ }
73
+
74
+ r.cache.Add(name, cacheEntry{
75
+ val: val,
76
+ eol: cacheTil,
77
+ })
78
+}
79
+
80
+type cacheEntry struct {
81
+ val path.Path
82
+ eol time.Time
83
+}
84
+
85
+// NewRoutingResolver constructs a name resolver using the IPFS Routing system
86
+// to implement SFS-like naming on top.
87
+// cachesize is the limit of the number of entries in the lru cache. Setting it
88
+// to '0' will disable caching.
89
+func NewRoutingResolver(route routing.IpfsRouting, cachesize int) *routingResolver {
90
if route == nil {
91
panic("attempt to create resolver with nil routing system")
92
}
93
40
- return &routingResolver{routing: route}
94
+ var cache *lru.Cache
95
+ if cachesize > 0 {
96
+ cache, _ = lru.New(cachesize)
97
+ }
98
+
99
+ return &routingResolver{
100
+ routing: route,
101
+ cache: cache,
102
+ }
103
}
104
105
// Resolve implements Resolver.
@@ -54,6 +116,11 @@ func (r *routingResolver) ResolveN(ctx context.Context, name string, depth int)
116
// resolve SFS-like names.
117
func (r *routingResolver) resolveOnce(ctx context.Context, name string) (path.Path, error) {
118
log.Debugf("RoutingResolve: '%s'", name)
119
+ cached, ok := r.cacheGet(name)
120
+ if ok {
121
+ return cached, nil
122
+ }
123
+
124
hash, err := mh.FromB58String(name)
125
if err != nil {
126
log.Warning("RoutingResolve: bad input hash: [%s]\n", name)
@@ -98,10 +165,29 @@ func (r *routingResolver) resolveOnce(ctx context.Context, name string) (path.Pa
165
valh, err := mh.Cast(entry.GetValue())
166
if err != nil {
167
// Not a multihash, probably a new record
101
- return path.ParsePath(string(entry.GetValue()))
168
+ p, err := path.ParsePath(string(entry.GetValue()))
169
+ if err != nil {
170
+ return "", err
171
+ }
172
+
173
+ r.cacheSet(name, p, entry)
174
+ return p, nil
175
} else {
176
// Its an old style multihash record
177
log.Warning("Detected old style multihash record")
105
- return path.FromKey(key.Key(valh)), nil
178
+ p := path.FromKey(key.Key(valh))
179
+ r.cacheSet(name, p, entry)
180
+ return p, nil
181
+ }
182
+}
183
+
184
+func checkEOL(e *pb.IpnsEntry) (time.Time, bool) {
185
+ if e.GetValidityType() == pb.IpnsEntry_EOL {
186
+ eol, err := u.ParseRFC3339(string(e.GetValidity()))
187
+ if err != nil {
188
+ return time.Time{}, false
189
+ }
190
+ return eol, true
191
}
192
+ return time.Time{}, false
193
}
repo/config/init.go
+4
@@ -64,6 +64,10 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) {
64
IPNS: "/ipns",
65
},
66
67
+ Ipns: Ipns{
68
+ ResolveCacheSize: 128,
69
+ },
70
+
71
// tracking ipfs version used to generate the init folder and adding
72
// update checker default setting.
73
Version: VersionDefaultValue(),
repo/config/ipns.go
+2
@@ -3,4 +3,6 @@ package config
3
type Ipns struct {
4
RepublishPeriod string
5
RecordLifetime string
6
+
7
+ ResolveCacheSize int
8
}
test/sharness/t0240-republisher.sh
+1
@@ -26,6 +26,7 @@ setup_iptb() {
26
for i in $(test_seq 0 3)
27
do
28
ipfsi $i config Ipns.RepublishPeriod 20s
29
+ ipfsi $i config --json Ipns.ResolveCacheSize 0
30
done
31
'
32