namesys: Implement async methods
License: MIT Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
Łukasz Magiera committed
Aug 28, 2018 at 00:44 UTC
7ff9f09b07a493fdcadaba6f6954519514e04fa6
8 files changed
+390
-22
core/commands/name/ipns.go
+2
@@ -30,6 +30,7 @@ const (
30
nocacheOptionName = "nocache"
31
dhtRecordCountOptionName = "dht-record-count"
32
dhtTimeoutOptionName = "dht-timeout"
33
+ streamOptionName = "stream"
34
)
35
36
var IpnsCmd = &cmds.Command{
@@ -78,6 +79,7 @@ Resolve the value of a dnslink:
79
cmdkit.BoolOption(nocacheOptionName, "n", "Do not use cached entries."),
80
cmdkit.UintOption(dhtRecordCountOptionName, "dhtrc", "Number of records to request for DHT resolution."),
81
cmdkit.StringOption(dhtTimeoutOptionName, "dhtt", "Max time to collect values during DHT resolution eg \"30s\". Pass 0 for no timeout."),
82
+ cmdkit.BoolOption(streamOptionName, "s", "Stream entries as they are found."),
83
},
84
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
85
n, err := cmdenv.GetNode(env)
namesys/base.go
+81
-14
@@ -10,13 +10,21 @@ import (
10
path "gx/ipfs/QmcjwUb36Z16NJkvDX6ccXPqsFswo6AsRXynyXcLLCphV2/go-path"
11
)
12
13
+type onceResult struct {
14
+ value path.Path
15
+ ttl time.Duration
16
+ err error
17
+}
18
+
19
type resolver interface {
20
// resolveOnce looks up a name once (without recursion).
15
- resolveOnce(ctx context.Context, name string, options *opts.ResolveOpts) (value path.Path, ttl time.Duration, err error)
21
+ resolveOnce(ctx context.Context, name string, options opts.ResolveOpts) (value path.Path, ttl time.Duration, err error)
22
+
23
+ resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult
24
}
25
26
// resolve is a helper for implementing Resolver.ResolveN using resolveOnce.
19
-func resolve(ctx context.Context, r resolver, name string, options *opts.ResolveOpts, prefixes ...string) (path.Path, error) {
27
+func resolve(ctx context.Context, r resolver, name string, options opts.ResolveOpts, prefix string) (path.Path, error) {
28
depth := options.Depth
29
for {
30
p, _, err := r.resolveOnce(ctx, name, options)
@@ -34,23 +42,82 @@ func resolve(ctx context.Context, r resolver, name string, options *opts.Resolve
42
return p, ErrResolveRecursion
43
}
44
37
- matched := false
38
- for _, prefix := range prefixes {
39
- if strings.HasPrefix(p.String(), prefix) {
40
- matched = true
41
- if len(prefixes) == 1 {
42
- name = strings.TrimPrefix(p.String(), prefix)
43
- }
44
- break
45
- }
46
- }
47
-
48
- if !matched {
45
+ if !strings.HasPrefix(p.String(), prefix) {
46
return p, nil
47
}
48
+ name = strings.TrimPrefix(p.String(), prefix)
49
50
if depth > 1 {
51
depth--
52
}
53
}
54
}
55
+
56
+//TODO:
57
+// - better error handling
58
+func resolveAsyncDo(ctx context.Context, r resolver, name string, options opts.ResolveOpts, prefix string) <-chan Result {
59
+ resCh := r.resolveOnceAsync(ctx, name, options)
60
+ depth := options.Depth
61
+ outCh := make(chan Result)
62
+
63
+ go func() {
64
+ defer close(outCh)
65
+ var subCh <-chan Result
66
+ var cancelSub context.CancelFunc
67
+
68
+ for {
69
+ select {
70
+ case res, ok := <-resCh:
71
+ if res.err != nil {
72
+ outCh <- Result{err: res.err}
73
+ return
74
+ }
75
+ if !ok {
76
+ resCh = nil
77
+ continue
78
+ }
79
+ log.Debugf("resolved %s to %s", name, res.value.String())
80
+ if strings.HasPrefix(res.value.String(), "/ipfs/") {
81
+ outCh <- Result{err: res.err}
82
+ continue
83
+ }
84
+ p := strings.TrimPrefix(res.value.String(), prefix)
85
+
86
+ if depth == 1 {
87
+ outCh <- Result{err: ErrResolveRecursion}
88
+ continue
89
+ }
90
+
91
+ subopts := options
92
+ if subopts.Depth > 1 {
93
+ subopts.Depth--
94
+ }
95
+
96
+ var subCtx context.Context
97
+ if subCh != nil {
98
+ // Cancel previous recursive resolve since it won't be used anyways
99
+ cancelSub()
100
+ }
101
+ subCtx, cancelSub = context.WithCancel(ctx)
102
+
103
+ subCh = resolveAsyncDo(subCtx, r, p, subopts, prefix)
104
+ case res, ok := <-subCh:
105
+ if res.err != nil {
106
+ outCh <- Result{err: res.err}
107
+ return
108
+ }
109
+ if !ok {
110
+ subCh = nil
111
+ continue
112
+ }
113
+ outCh <- res
114
+ case <-ctx.Done():
115
+ }
116
+ }
117
+ }()
118
+ return outCh
119
+}
120
+
121
+func resolveAsync(ctx context.Context, r resolver, name string, options opts.ResolveOpts, prefix string) <-chan Result {
122
+ return resolveAsyncDo(ctx, r, name, options, prefix)
123
+}
namesys/dns.go
+56
-1
@@ -39,7 +39,7 @@ type lookupRes struct {
39
// resolveOnce implements resolver.
40
// TXT records for a given domain name should contain a b58
41
// encoded multihash.
42
-func (r *DNSResolver) resolveOnce(ctx context.Context, name string, options *opts.ResolveOpts) (path.Path, time.Duration, error) {
42
+func (r *DNSResolver) resolveOnce(ctx context.Context, name string, options opts.ResolveOpts) (path.Path, time.Duration, error) {
43
segments := strings.SplitN(name, "/", 2)
44
domain := segments[0]
45
@@ -84,6 +84,61 @@ func (r *DNSResolver) resolveOnce(ctx context.Context, name string, options *opt
84
return p, 0, err
85
}
86
87
+func (r *DNSResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult {
88
+ out := make(chan onceResult, 1)
89
+ segments := strings.SplitN(name, "/", 2)
90
+ domain := segments[0]
91
+
92
+ if !isd.IsDomain(domain) {
93
+ out <- onceResult{err: errors.New("not a valid domain name")}
94
+ close(out)
95
+ return out
96
+ }
97
+ log.Debugf("DNSResolver resolving %s", domain)
98
+
99
+ rootChan := make(chan lookupRes, 1)
100
+ go workDomain(r, domain, rootChan)
101
+
102
+ subChan := make(chan lookupRes, 1)
103
+ go workDomain(r, "_dnslink."+domain, subChan)
104
+
105
+ go func() {
106
+ defer close(out)
107
+ for {
108
+ select {
109
+ case subRes, ok := <-subChan:
110
+ if !ok {
111
+ subChan = nil
112
+ }
113
+ if subRes.error == nil {
114
+ select {
115
+ case out <- onceResult{value: subRes.path}:
116
+ case <-ctx.Done():
117
+ }
118
+ return
119
+ }
120
+ case rootRes, ok := <-rootChan:
121
+ if !ok {
122
+ subChan = nil
123
+ }
124
+ if rootRes.error == nil {
125
+ select {
126
+ case out <- onceResult{value: rootRes.path}:
127
+ case <-ctx.Done():
128
+ }
129
+ }
130
+ case <-ctx.Done():
131
+ return
132
+ }
133
+ if subChan == nil && rootChan == nil {
134
+ return
135
+ }
136
+ }
137
+ }()
138
+
139
+ return out
140
+}
141
+
142
func workDomain(r *DNSResolver, name string, res chan lookupRes) {
143
txt, err := r.lookupTXT(name)
144
namesys/interface.go
+11
@@ -63,6 +63,12 @@ type NameSystem interface {
63
Publisher
64
}
65
66
+// Result is the return type for Resolver.ResolveAsync.
67
+type Result struct {
68
+ path path.Path
69
+ err error
70
+}
71
+
72
// Resolver is an object capable of resolving names.
73
type Resolver interface {
74
@@ -81,6 +87,11 @@ type Resolver interface {
87
// users will be fine with this default limit, but if you need to
88
// adjust the limit you can specify it as an option.
89
Resolve(ctx context.Context, name string, options ...opts.ResolveOpt) (value path.Path, err error)
90
+
91
+ // ResolveAsync performs recursive name lookup, like Resolve, but it returns
92
+ // entries as they are discovered in the DHT. Each returned result is guaranteed
93
+ // to be "better" (which usually means newer) than the previous one.
94
+ ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result
95
}
96
97
// Publisher is an object capable of publishing particular names.
namesys/namesys.go
+87
-1
@@ -64,8 +64,25 @@ func (ns *mpns) Resolve(ctx context.Context, name string, options ...opts.Resolv
64
return resolve(ctx, ns, name, opts.ProcessOpts(options), "/ipns/")
65
}
66
67
+func (ns *mpns) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result {
68
+ res := make(chan Result, 1)
69
+ if strings.HasPrefix(name, "/ipfs/") {
70
+ p, err := path.ParsePath(name)
71
+ res <- Result{p, err}
72
+ return res
73
+ }
74
+
75
+ if !strings.HasPrefix(name, "/") {
76
+ p, err := path.ParsePath("/ipfs/" + name)
77
+ res <- Result{p, err}
78
+ return res
79
+ }
80
+
81
+ return resolveAsync(ctx, ns, name, opts.ProcessOpts(options), "/ipns/")
82
+}
83
+
84
// resolveOnce implements resolver.
68
-func (ns *mpns) resolveOnce(ctx context.Context, name string, options *opts.ResolveOpts) (path.Path, time.Duration, error) {
85
+func (ns *mpns) resolveOnce(ctx context.Context, name string, options opts.ResolveOpts) (path.Path, time.Duration, error) {
86
if !strings.HasPrefix(name, "/ipns/") {
87
name = "/ipns/" + name
88
}
@@ -107,6 +124,75 @@ func (ns *mpns) resolveOnce(ctx context.Context, name string, options *opts.Reso
124
return p, 0, err
125
}
126
127
+func (ns *mpns) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult {
128
+ out := make(chan onceResult, 1)
129
+
130
+ if !strings.HasPrefix(name, "/ipns/") {
131
+ name = "/ipns/" + name
132
+ }
133
+ segments := strings.SplitN(name, "/", 4)
134
+ if len(segments) < 3 || segments[0] != "" {
135
+ log.Debugf("invalid name syntax for %s", name)
136
+ out <- onceResult{err: ErrResolveFailed}
137
+ close(out)
138
+ return out
139
+ }
140
+
141
+ key := segments[2]
142
+
143
+ if p, ok := ns.cacheGet(key); ok {
144
+ out <- onceResult{value: p}
145
+ close(out)
146
+ return out
147
+ }
148
+
149
+ // Resolver selection:
150
+ // 1. if it is a multihash resolve through "ipns".
151
+ // 2. if it is a domain name, resolve through "dns"
152
+ // 3. otherwise resolve through the "proquint" resolver
153
+
154
+ var res resolver
155
+ if _, err := mh.FromB58String(key); err == nil {
156
+ res = ns.ipnsResolver
157
+ } else if isd.IsDomain(key) {
158
+ res = ns.dnsResolver
159
+ } else {
160
+ res = ns.proquintResolver
161
+ }
162
+
163
+ resCh := res.resolveOnceAsync(ctx, key, options)
164
+ var best onceResult
165
+ go func() {
166
+ defer close(out)
167
+ for {
168
+ select {
169
+ case res, ok := <-resCh:
170
+ if !ok {
171
+ if best != (onceResult{}) {
172
+ ns.cacheSet(key, best.value, best.ttl)
173
+ }
174
+ return
175
+ }
176
+ if res.err == nil {
177
+ best = res
178
+ }
179
+ p := res.value
180
+
181
+ // Attach rest of the path
182
+ if len(segments) > 3 {
183
+ p, _ = path.FromSegments("", strings.TrimRight(p.String(), "/"), segments[3])
184
+ }
185
+
186
+ out <- onceResult{value: p, err: res.err}
187
+ case <-ctx.Done():
188
+ return
189
+ }
190
+ }
191
+ }()
192
+
193
+ return out
194
+}
195
+
196
// Publish implements Publisher
197
func (ns *mpns) Publish(ctx context.Context, name ci.PrivKey, value path.Path) error {
198
return ns.PublishWithEOL(ctx, name, value, time.Now().Add(DefaultRecordTTL))
namesys/opts/opts.go
+4
-4
@@ -31,8 +31,8 @@ type ResolveOpts struct {
31
32
// DefaultResolveOpts returns the default options for resolving
33
// an IPNS path
34
-func DefaultResolveOpts() *ResolveOpts {
35
- return &ResolveOpts{
34
+func DefaultResolveOpts() ResolveOpts {
35
+ return ResolveOpts{
36
Depth: DefaultDepthLimit,
37
DhtRecordCount: 16,
38
DhtTimeout: time.Minute,
@@ -65,10 +65,10 @@ func DhtTimeout(timeout time.Duration) ResolveOpt {
65
}
66
67
// ProcessOpts converts an array of ResolveOpt into a ResolveOpts object
68
-func ProcessOpts(opts []ResolveOpt) *ResolveOpts {
68
+func ProcessOpts(opts []ResolveOpt) ResolveOpts {
69
rsopts := DefaultResolveOpts()
70
for _, option := range opts {
71
- option(rsopts)
71
+ option(&rsopts)
72
}
73
return rsopts
74
}
namesys/proquint.go
+15
-1
@@ -19,7 +19,7 @@ func (r *ProquintResolver) Resolve(ctx context.Context, name string, options ...
19
}
20
21
// resolveOnce implements resolver. Decodes the proquint string.
22
-func (r *ProquintResolver) resolveOnce(ctx context.Context, name string, options *opts.ResolveOpts) (path.Path, time.Duration, error) {
22
+func (r *ProquintResolver) resolveOnce(ctx context.Context, name string, options opts.ResolveOpts) (path.Path, time.Duration, error) {
23
ok, err := proquint.IsProquint(name)
24
if err != nil || !ok {
25
return "", 0, errors.New("not a valid proquint string")
@@ -27,3 +27,17 @@ func (r *ProquintResolver) resolveOnce(ctx context.Context, name string, options
27
// Return a 0 TTL as caching this result is pointless.
28
return path.FromString(string(proquint.Decode(name))), 0, nil
29
}
30
+
31
+func (r *ProquintResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult {
32
+ out := make(chan onceResult, 1)
33
+ defer close(out)
34
+
35
+ ok, err := proquint.IsProquint(name)
36
+ if err != nil || !ok {
37
+ out <- onceResult{err: errors.New("not a valid proquint string")}
38
+ return out
39
+ }
40
+ // Return a 0 TTL as caching this result is pointless.
41
+ out <- onceResult{value: path.FromString(string(proquint.Decode(name)))}
42
+ return out
43
+}
namesys/routing.go
+134
-1
@@ -42,9 +42,13 @@ func (r *IpnsResolver) Resolve(ctx context.Context, name string, options ...opts
42
return resolve(ctx, r, name, opts.ProcessOpts(options), "/ipns/")
43
}
44
45
+func (r *IpnsResolver) ResolveAsync(ctx context.Context, name string, options ...opts.ResolveOpt) <-chan Result {
46
+ return resolveAsync(ctx, r, name, opts.ProcessOpts(options), "/ipns/")
47
+}
48
+
49
// resolveOnce implements resolver. Uses the IPFS routing system to
50
// resolve SFS-like names.
47
-func (r *IpnsResolver) resolveOnce(ctx context.Context, name string, options *opts.ResolveOpts) (path.Path, time.Duration, error) {
51
+func (r *IpnsResolver) resolveOnce(ctx context.Context, name string, options opts.ResolveOpts) (path.Path, time.Duration, error) {
52
log.Debugf("RoutingResolver resolving %s", name)
53
54
if options.DhtTimeout != 0 {
@@ -126,3 +130,132 @@ func (r *IpnsResolver) resolveOnce(ctx context.Context, name string, options *op
130
131
return p, ttl, nil
132
}
133
+
134
+func (r *IpnsResolver) resolveOnceAsync(ctx context.Context, name string, options opts.ResolveOpts) <-chan onceResult {
135
+ out := make(chan onceResult, 1)
136
+ log.Debugf("RoutingResolver resolving %s", name)
137
+ if options.DhtTimeout != 0 {
138
+ // Resolution must complete within the timeout
139
+ var cancel context.CancelFunc
140
+ ctx, cancel = context.WithTimeout(ctx, options.DhtTimeout)
141
+ defer cancel()
142
+ }
143
+
144
+ name = strings.TrimPrefix(name, "/ipns/")
145
+ hash, err := mh.FromB58String(name)
146
+ if err != nil {
147
+ // name should be a multihash. if it isn't, error out here.
148
+ log.Debugf("RoutingResolver: bad input hash: [%s]\n", name)
149
+ out <- onceResult{err: err}
150
+ close(out)
151
+ return out
152
+ }
153
+
154
+ pid, err := peer.IDFromBytes(hash)
155
+ if err != nil {
156
+ log.Debugf("RoutingResolver: could not convert public key hash %s to peer ID: %s\n", name, err)
157
+ out <- onceResult{err: err}
158
+ close(out)
159
+ return out
160
+ }
161
+
162
+ // Name should be the hash of a public key retrievable from ipfs.
163
+ // We retrieve the public key here to make certain that it's in the peer
164
+ // store before calling GetValue() on the DHT - the DHT will call the
165
+ // ipns validator, which in turn will get the public key from the peer
166
+ // store to verify the record signature
167
+ _, err = routing.GetPublicKey(r.routing, ctx, pid)
168
+ if err != nil {
169
+ log.Debugf("RoutingResolver: could not retrieve public key %s: %s\n", name, err)
170
+ out <- onceResult{err: err}
171
+ close(out)
172
+ return out
173
+ }
174
+
175
+ // Use the routing system to get the name.
176
+ // Note that the DHT will call the ipns validator when retrieving
177
+ // the value, which in turn verifies the ipns record signature
178
+ ipnsKey := ipns.RecordKey(pid)
179
+
180
+ vals, err := r.routing.(*dht.IpfsDHT).SearchValue(ctx, ipnsKey, dht.Quorum(int(options.DhtRecordCount)))
181
+ if err != nil {
182
+ log.Debugf("RoutingResolver: dht get for name %s failed: %s", name, err)
183
+ out <- onceResult{err: err}
184
+ close(out)
185
+ return out
186
+ }
187
+
188
+ go func() {
189
+ defer close(out)
190
+ for {
191
+ select {
192
+ case val, ok := <-vals:
193
+ if !ok {
194
+ return
195
+ }
196
+
197
+ entry := new(pb.IpnsEntry)
198
+ err = proto.Unmarshal(val, entry)
199
+ if err != nil {
200
+ log.Debugf("RoutingResolver: could not unmarshal value for name %s: %s", name, err)
201
+ select {
202
+ case out <- onceResult{err: err}:
203
+ case <-ctx.Done():
204
+ }
205
+ return
206
+ }
207
+
208
+ var p path.Path
209
+ // check for old style record:
210
+ if valh, err := mh.Cast(entry.GetValue()); err == nil {
211
+ // Its an old style multihash record
212
+ log.Debugf("encountered CIDv0 ipns entry: %s", valh)
213
+ p = path.FromCid(cid.NewCidV0(valh))
214
+ } else {
215
+ // Not a multihash, probably a new style record
216
+ p, err = path.ParsePath(string(entry.GetValue()))
217
+ if err != nil {
218
+ select {
219
+ case out <- onceResult{err: err}:
220
+ case <-ctx.Done():
221
+ }
222
+ return
223
+ }
224
+ }
225
+
226
+ ttl := DefaultResolverCacheTTL
227
+ if entry.Ttl != nil {
228
+ ttl = time.Duration(*entry.Ttl)
229
+ }
230
+ switch eol, err := ipns.GetEOL(entry); err {
231
+ case ipns.ErrUnrecognizedValidity:
232
+ // No EOL.
233
+ case nil:
234
+ ttEol := eol.Sub(time.Now())
235
+ if ttEol < 0 {
236
+ // It *was* valid when we first resolved it.
237
+ ttl = 0
238
+ } else if ttEol < ttl {
239
+ ttl = ttEol
240
+ }
241
+ default:
242
+ log.Errorf("encountered error when parsing EOL: %s", err)
243
+ select {
244
+ case out <- onceResult{err: err}:
245
+ case <-ctx.Done():
246
+ }
247
+ return
248
+ }
249
+
250
+ select {
251
+ case out <- onceResult{value: p, ttl: ttl}:
252
+ case <-ctx.Done():
253
+ }
254
+ case <-ctx.Done():
255
+ return
256
+ }
257
+ }
258
+ }()
259
+
260
+ return out
261
+}