@cryptotaxi247 / kubo / commits / acfc35212

add backoff retry to reprovider

Jeromy committed Jan 17, 2015 at 04:06 UTC acfc35212ea7c33e69b3c04f4a27c0da5d6d6e03
15 files changed +699 -3
Godeps/Godeps.json
+4
@@ -60,6 +60,10 @@
60 "ImportPath": "github.com/camlistore/lock",
61 "Rev": "ae27720f340952636b826119b58130b9c1a847a0"
62 },
63 + {
64 + "ImportPath": "github.com/cenkalti/backoff",
65 + "Rev": "9831e1e25c874e0a0601b6dc43641071414eec7a"
66 + },
67 {
68 "ImportPath": "github.com/coreos/go-semver/semver",
69 "Rev": "6fe83ccda8fb9b7549c9ab4ba47f47858bc950aa"
Godeps/_workspace/src/github.com/cenkalti/backoff/.gitignore new
+22
@@ -0,0 +1,22 @@
1 +# Compiled Object files, Static and Dynamic libs (Shared Objects)
2 +*.o
3 +*.a
4 +*.so
5 +
6 +# Folders
7 +_obj
8 +_test
9 +
10 +# Architecture specific extensions/prefixes
11 +*.[568vq]
12 +[568vq].out
13 +
14 +*.cgo1.go
15 +*.cgo2.c
16 +_cgo_defun.c
17 +_cgo_gotypes.go
18 +_cgo_export.*
19 +
20 +_testmain.go
21 +
22 +*.exe
Godeps/_workspace/src/github.com/cenkalti/backoff/.travis.yml new
+2
@@ -0,0 +1,2 @@
1 +language: go
2 +go: 1.3.3
Godeps/_workspace/src/github.com/cenkalti/backoff/LICENSE new
+20
@@ -0,0 +1,20 @@
1 +The MIT License (MIT)
2 +
3 +Copyright (c) 2014 Cenk Altı
4 +
5 +Permission is hereby granted, free of charge, to any person obtaining a copy of
6 +this software and associated documentation files (the "Software"), to deal in
7 +the Software without restriction, including without limitation the rights to
8 +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
9 +the Software, and to permit persons to whom the Software is furnished to do so,
10 +subject to the following conditions:
11 +
12 +The above copyright notice and this permission notice shall be included in all
13 +copies or substantial portions of the Software.
14 +
15 +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17 +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18 +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19 +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20 +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Godeps/_workspace/src/github.com/cenkalti/backoff/README.md new
+69
@@ -0,0 +1,69 @@
1 +# backoff
2 +
3 +[![GoDoc](https://godoc.org/github.com/cenkalti/backoff?status.png)](https://godoc.org/github.com/cenkalti/backoff)
4 +[![Build Status](https://travis-ci.org/cenkalti/backoff.png)](https://travis-ci.org/cenkalti/backoff)
5 +
6 +This is a Go port of the exponential backoff algorithm from
7 +[google-http-java-client](https://code.google.com/p/google-http-java-client/wiki/ExponentialBackoff).
8 +
9 +[Exponential backoff](http://en.wikipedia.org/wiki/Exponential_backoff)
10 +is an algorithm that uses feedback to multiplicatively decrease the rate of some process,
11 +in order to gradually find an acceptable rate.
12 +The retries exponentially increase and stop increasing when a certain threshold is met.
13 +
14 +
15 +
16 +
17 +## Install
18 +
19 +```bash
20 +go get github.com/cenkalti/backoff
21 +```
22 +
23 +## Example
24 +
25 +Simple retry helper that uses exponential back-off algorithm:
26 +
27 +```go
28 +operation := func() error {
29 + // An operation that might fail
30 +}
31 +
32 +err := backoff.Retry(operation, backoff.NewExponentialBackOff())
33 +if err != nil {
34 + // handle error
35 +}
36 +
37 +// operation is successfull
38 +```
39 +
40 +Ticker example:
41 +
42 +```go
43 +operation := func() error {
44 + // An operation that may fail
45 +}
46 +
47 +b := backoff.NewExponentialBackOff()
48 +ticker := backoff.NewTicker(b)
49 +
50 +var err error
51 +
52 +// Ticks will continue to arrive when the previous operation is still running,
53 +// so operations that take a while to fail could run in quick succession.
54 +for t = range ticker.C {
55 + if err = operation(); err != nil {
56 + log.Println(err, "will retry...")
57 + continue
58 + }
59 +
60 + ticker.Stop()
61 + break
62 +}
63 +
64 +if err != nil {
65 + // Operation has failed.
66 +}
67 +
68 +// Operation is successfull.
69 +```
Godeps/_workspace/src/github.com/cenkalti/backoff/backoff.go new
+56
@@ -0,0 +1,56 @@
1 +// Package backoff implements backoff algorithms for retrying operations.
2 +//
3 +// Also has a Retry() helper for retrying operations that may fail.
4 +package backoff
5 +
6 +import "time"
7 +
8 +// Back-off policy when retrying an operation.
9 +type BackOff interface {
10 + // Gets the duration to wait before retrying the operation or
11 + // backoff.Stop to indicate that no retries should be made.
12 + //
13 + // Example usage:
14 + //
15 + // duration := backoff.NextBackOff();
16 + // if (duration == backoff.Stop) {
17 + // // do not retry operation
18 + // } else {
19 + // // sleep for duration and retry operation
20 + // }
21 + //
22 + NextBackOff() time.Duration
23 +
24 + // Reset to initial state.
25 + Reset()
26 +}
27 +
28 +// Indicates that no more retries should be made for use in NextBackOff().
29 +const Stop time.Duration = -1
30 +
31 +// ZeroBackOff is a fixed back-off policy whose back-off time is always zero,
32 +// meaning that the operation is retried immediately without waiting.
33 +type ZeroBackOff struct{}
34 +
35 +func (b *ZeroBackOff) Reset() {}
36 +
37 +func (b *ZeroBackOff) NextBackOff() time.Duration { return 0 }
38 +
39 +// StopBackOff is a fixed back-off policy that always returns backoff.Stop for
40 +// NextBackOff(), meaning that the operation should not be retried.
41 +type StopBackOff struct{}
42 +
43 +func (b *StopBackOff) Reset() {}
44 +
45 +func (b *StopBackOff) NextBackOff() time.Duration { return Stop }
46 +
47 +type ConstantBackOff struct {
48 + Interval time.Duration
49 +}
50 +
51 +func (b *ConstantBackOff) Reset() {}
52 +func (b *ConstantBackOff) NextBackOff() time.Duration { return b.Interval }
53 +
54 +func NewConstantBackOff(d time.Duration) *ConstantBackOff {
55 + return &ConstantBackOff{Interval: d}
56 +}
Godeps/_workspace/src/github.com/cenkalti/backoff/backoff_test.go new
+28
@@ -0,0 +1,28 @@
1 +package backoff
2 +
3 +import (
4 + "time"
5 +
6 + "testing"
7 +)
8 +
9 +func TestNextBackOffMillis(t *testing.T) {
10 + subtestNextBackOff(t, 0, new(ZeroBackOff))
11 + subtestNextBackOff(t, Stop, new(StopBackOff))
12 +}
13 +
14 +func subtestNextBackOff(t *testing.T, expectedValue time.Duration, backOffPolicy BackOff) {
15 + for i := 0; i < 10; i++ {
16 + next := backOffPolicy.NextBackOff()
17 + if next != expectedValue {
18 + t.Errorf("got: %d expected: %d", next, expectedValue)
19 + }
20 + }
21 +}
22 +
23 +func TestConstantBackOff(t *testing.T) {
24 + backoff := NewConstantBackOff(time.Second)
25 + if backoff.NextBackOff() != time.Second {
26 + t.Error("invalid interval")
27 + }
28 +}
Godeps/_workspace/src/github.com/cenkalti/backoff/exponential.go new
+141
@@ -0,0 +1,141 @@
1 +package backoff
2 +
3 +import (
4 + "math/rand"
5 + "time"
6 +)
7 +
8 +/*
9 +ExponentialBackOff is an implementation of BackOff that increases the back off
10 +period for each retry attempt using a randomization function that grows exponentially.
11 +
12 +NextBackOff() is calculated using the following formula:
13 +
14 + randomized_interval =
15 + retry_interval * (random value in range [1 - randomization_factor, 1 + randomization_factor])
16 +
17 +In other words NextBackOff() will range between the randomization factor
18 +percentage below and above the retry interval. For example, using 2 seconds as the base retry
19 +interval and 0.5 as the randomization factor, the actual back off period used in the next retry
20 +attempt will be between 1 and 3 seconds.
21 +
22 +Note: max_interval caps the retry_interval and not the randomized_interval.
23 +
24 +If the time elapsed since an ExponentialBackOff instance is created goes past the
25 +max_elapsed_time then the method NextBackOff() starts returning backoff.Stop.
26 +The elapsed time can be reset by calling Reset().
27 +
28 +Example: The default retry_interval is .5 seconds, default randomization_factor is 0.5, default
29 +multiplier is 1.5 and the default max_interval is 1 minute. For 10 tries the sequence will be
30 +(values in seconds) and assuming we go over the max_elapsed_time on the 10th try:
31 +
32 + request# retry_interval randomized_interval
33 +
34 + 1 0.5 [0.25, 0.75]
35 + 2 0.75 [0.375, 1.125]
36 + 3 1.125 [0.562, 1.687]
37 + 4 1.687 [0.8435, 2.53]
38 + 5 2.53 [1.265, 3.795]
39 + 6 3.795 [1.897, 5.692]
40 + 7 5.692 [2.846, 8.538]
41 + 8 8.538 [4.269, 12.807]
42 + 9 12.807 [6.403, 19.210]
43 + 10 19.210 backoff.Stop
44 +
45 +Implementation is not thread-safe.
46 +*/
47 +type ExponentialBackOff struct {
48 + InitialInterval time.Duration
49 + RandomizationFactor float64
50 + Multiplier float64
51 + MaxInterval time.Duration
52 + // After MaxElapsedTime the ExponentialBackOff stops.
53 + // It never stops if MaxElapsedTime == 0.
54 + MaxElapsedTime time.Duration
55 + Clock Clock
56 +
57 + currentInterval time.Duration
58 + startTime time.Time
59 +}
60 +
61 +// Clock is an interface that returns current time for BackOff.
62 +type Clock interface {
63 + Now() time.Time
64 +}
65 +
66 +// Default values for ExponentialBackOff.
67 +const (
68 + DefaultInitialInterval = 500 * time.Millisecond
69 + DefaultRandomizationFactor = 0.5
70 + DefaultMultiplier = 1.5
71 + DefaultMaxInterval = 60 * time.Second
72 + DefaultMaxElapsedTime = 15 * time.Minute
73 +)
74 +
75 +// NewExponentialBackOff creates an instance of ExponentialBackOff using default values.
76 +func NewExponentialBackOff() *ExponentialBackOff {
77 + return &ExponentialBackOff{
78 + InitialInterval: DefaultInitialInterval,
79 + RandomizationFactor: DefaultRandomizationFactor,
80 + Multiplier: DefaultMultiplier,
81 + MaxInterval: DefaultMaxInterval,
82 + MaxElapsedTime: DefaultMaxElapsedTime,
83 + Clock: SystemClock,
84 + }
85 +}
86 +
87 +type systemClock struct{}
88 +
89 +func (t systemClock) Now() time.Time {
90 + return time.Now()
91 +}
92 +
93 +// SystemClock implements Clock interface that uses time.Now().
94 +var SystemClock = systemClock{}
95 +
96 +// Reset the interval back to the initial retry interval and restarts the timer.
97 +func (b *ExponentialBackOff) Reset() {
98 + b.currentInterval = b.InitialInterval
99 + b.startTime = b.Clock.Now()
100 +}
101 +
102 +// NextBackOff calculates the next back off interval using the formula:
103 +// randomized_interval = retry_interval +/- (randomization_factor * retry_interval)
104 +func (b *ExponentialBackOff) NextBackOff() time.Duration {
105 + // Make sure we have not gone over the maximum elapsed time.
106 + if b.MaxElapsedTime != 0 && b.GetElapsedTime() > b.MaxElapsedTime {
107 + return Stop
108 + }
109 + defer b.incrementCurrentInterval()
110 + return getRandomValueFromInterval(b.RandomizationFactor, rand.Float64(), b.currentInterval)
111 +}
112 +
113 +// GetElapsedTime returns the elapsed time since an ExponentialBackOff instance
114 +// is created and is reset when Reset() is called.
115 +//
116 +// The elapsed time is computed using time.Now().UnixNano().
117 +func (b *ExponentialBackOff) GetElapsedTime() time.Duration {
118 + return b.Clock.Now().Sub(b.startTime)
119 +}
120 +
121 +// Increments the current interval by multiplying it with the multiplier.
122 +func (b *ExponentialBackOff) incrementCurrentInterval() {
123 + // Check for overflow, if overflow is detected set the current interval to the max interval.
124 + if float64(b.currentInterval) >= float64(b.MaxInterval)/b.Multiplier {
125 + b.currentInterval = b.MaxInterval
126 + } else {
127 + b.currentInterval = time.Duration(float64(b.currentInterval) * b.Multiplier)
128 + }
129 +}
130 +
131 +// Returns a random value from the interval:
132 +// [randomizationFactor * currentInterval, randomizationFactor * currentInterval].
133 +func getRandomValueFromInterval(randomizationFactor, random float64, currentInterval time.Duration) time.Duration {
134 + var delta = randomizationFactor * float64(currentInterval)
135 + var minInterval = float64(currentInterval) - delta
136 + var maxInterval = float64(currentInterval) + delta
137 + // Get a random value from the range [minInterval, maxInterval].
138 + // The formula used below has a +1 because if the minInterval is 1 and the maxInterval is 3 then
139 + // we want a 33% chance for selecting either 1, 2 or 3.
140 + return time.Duration(minInterval + (random * (maxInterval - minInterval + 1)))
141 +}
Godeps/_workspace/src/github.com/cenkalti/backoff/exponential_test.go new
+111
@@ -0,0 +1,111 @@
1 +package backoff
2 +
3 +import (
4 + "math"
5 + "testing"
6 + "time"
7 +)
8 +
9 +func TestBackOff(t *testing.T) {
10 + var (
11 + testInitialInterval = 500 * time.Millisecond
12 + testRandomizationFactor = 0.1
13 + testMultiplier = 2.0
14 + testMaxInterval = 5 * time.Second
15 + testMaxElapsedTime = 15 * time.Minute
16 + )
17 +
18 + exp := NewExponentialBackOff()
19 + exp.InitialInterval = testInitialInterval
20 + exp.RandomizationFactor = testRandomizationFactor
21 + exp.Multiplier = testMultiplier
22 + exp.MaxInterval = testMaxInterval
23 + exp.MaxElapsedTime = testMaxElapsedTime
24 + exp.Reset()
25 +
26 + var expectedResults = []time.Duration{500, 1000, 2000, 4000, 5000, 5000, 5000, 5000, 5000, 5000}
27 + for i, d := range expectedResults {
28 + expectedResults[i] = d * time.Millisecond
29 + }
30 +
31 + for _, expected := range expectedResults {
32 + assertEquals(t, expected, exp.currentInterval)
33 + // Assert that the next back off falls in the expected range.
34 + var minInterval = expected - time.Duration(testRandomizationFactor*float64(expected))
35 + var maxInterval = expected + time.Duration(testRandomizationFactor*float64(expected))
36 + var actualInterval = exp.NextBackOff()
37 + if !(minInterval <= actualInterval && actualInterval <= maxInterval) {
38 + t.Error("error")
39 + }
40 + }
41 +}
42 +
43 +func TestGetRandomizedInterval(t *testing.T) {
44 + // 33% chance of being 1.
45 + assertEquals(t, 1, getRandomValueFromInterval(0.5, 0, 2))
46 + assertEquals(t, 1, getRandomValueFromInterval(0.5, 0.33, 2))
47 + // 33% chance of being 2.
48 + assertEquals(t, 2, getRandomValueFromInterval(0.5, 0.34, 2))
49 + assertEquals(t, 2, getRandomValueFromInterval(0.5, 0.66, 2))
50 + // 33% chance of being 3.
51 + assertEquals(t, 3, getRandomValueFromInterval(0.5, 0.67, 2))
52 + assertEquals(t, 3, getRandomValueFromInterval(0.5, 0.99, 2))
53 +}
54 +
55 +type TestClock struct {
56 + i time.Duration
57 + start time.Time
58 +}
59 +
60 +func (c *TestClock) Now() time.Time {
61 + t := c.start.Add(c.i)
62 + c.i += time.Second
63 + return t
64 +}
65 +
66 +func TestGetElapsedTime(t *testing.T) {
67 + var exp = NewExponentialBackOff()
68 + exp.Clock = &TestClock{}
69 + exp.Reset()
70 +
71 + var elapsedTime = exp.GetElapsedTime()
72 + if elapsedTime != time.Second {
73 + t.Errorf("elapsedTime=%d", elapsedTime)
74 + }
75 +}
76 +
77 +func TestMaxElapsedTime(t *testing.T) {
78 + var exp = NewExponentialBackOff()
79 + exp.Clock = &TestClock{start: time.Time{}.Add(10000 * time.Second)}
80 + if exp.NextBackOff() != Stop {
81 + t.Error("error2")
82 + }
83 + // Change the currentElapsedTime to be 0 ensuring that the elapsed time will be greater
84 + // than the max elapsed time.
85 + exp.startTime = time.Time{}
86 + assertEquals(t, Stop, exp.NextBackOff())
87 +}
88 +
89 +func TestBackOffOverflow(t *testing.T) {
90 + var (
91 + testInitialInterval time.Duration = math.MaxInt64 / 2
92 + testMaxInterval time.Duration = math.MaxInt64
93 + testMultiplier float64 = 2.1
94 + )
95 +
96 + exp := NewExponentialBackOff()
97 + exp.InitialInterval = testInitialInterval
98 + exp.Multiplier = testMultiplier
99 + exp.MaxInterval = testMaxInterval
100 + exp.Reset()
101 +
102 + exp.NextBackOff()
103 + // Assert that when an overflow is possible the current varerval time.Duration is set to the max varerval time.Duration .
104 + assertEquals(t, testMaxInterval, exp.currentInterval)
105 +}
106 +
107 +func assertEquals(t *testing.T, expected, value time.Duration) {
108 + if expected != value {
109 + t.Errorf("got: %d, expected: %d", value, expected)
110 + }
111 +}
Godeps/_workspace/src/github.com/cenkalti/backoff/retry.go new
+47
@@ -0,0 +1,47 @@
1 +package backoff
2 +
3 +import "time"
4 +
5 +// Retry the function f until it does not return error or BackOff stops.
6 +// f is guaranteed to be run at least once.
7 +// It is the caller's responsibility to reset b after Retry returns.
8 +//
9 +// Retry sleeps the goroutine for the duration returned by BackOff after a
10 +// failed operation returns.
11 +//
12 +// Usage:
13 +// operation := func() error {
14 +// // An operation that may fail
15 +// }
16 +//
17 +// err := backoff.Retry(operation, backoff.NewExponentialBackOff())
18 +// if err != nil {
19 +// // Operation has failed.
20 +// }
21 +//
22 +// // Operation is successfull.
23 +//
24 +func Retry(f func() error, b BackOff) error { return RetryNotify(f, b, nil) }
25 +
26 +// RetryNotify calls notify function with the error and wait duration for each failed attempt before sleep.
27 +func RetryNotify(f func() error, b BackOff, notify func(err error, wait time.Duration)) error {
28 + var err error
29 + var next time.Duration
30 +
31 + b.Reset()
32 + for {
33 + if err = f(); err == nil {
34 + return nil
35 + }
36 +
37 + if next = b.NextBackOff(); next == Stop {
38 + return err
39 + }
40 +
41 + if notify != nil {
42 + notify(err, next)
43 + }
44 +
45 + time.Sleep(next)
46 + }
47 +}
Godeps/_workspace/src/github.com/cenkalti/backoff/retry_test.go new
+34
@@ -0,0 +1,34 @@
1 +package backoff
2 +
3 +import (
4 + "errors"
5 + "log"
6 + "testing"
7 +)
8 +
9 +func TestRetry(t *testing.T) {
10 + const successOn = 3
11 + var i = 0
12 +
13 + // This function is successfull on "successOn" calls.
14 + f := func() error {
15 + i++
16 + log.Printf("function is called %d. time\n", i)
17 +
18 + if i == successOn {
19 + log.Println("OK")
20 + return nil
21 + }
22 +
23 + log.Println("error")
24 + return errors.New("error")
25 + }
26 +
27 + err := Retry(f, NewExponentialBackOff())
28 + if err != nil {
29 + t.Errorf("unexpected error: %s", err.Error())
30 + }
31 + if i != successOn {
32 + t.Errorf("invalid number of retries: %d", i)
33 + }
34 +}
Godeps/_workspace/src/github.com/cenkalti/backoff/ticker.go new
+105
@@ -0,0 +1,105 @@
1 +package backoff
2 +
3 +import (
4 + "runtime"
5 + "sync"
6 + "time"
7 +)
8 +
9 +// Ticker holds a channel that delivers `ticks' of a clock at times reported by a BackOff.
10 +//
11 +// Ticks will continue to arrive when the previous operation is still running,
12 +// so operations that take a while to fail could run in quick succession.
13 +//
14 +// Usage:
15 +// operation := func() error {
16 +// // An operation that may fail
17 +// }
18 +//
19 +// b := backoff.NewExponentialBackOff()
20 +// ticker := backoff.NewTicker(b)
21 +//
22 +// var err error
23 +// for _ = range ticker.C {
24 +// if err = operation(); err != nil {
25 +// log.Println(err, "will retry...")
26 +// continue
27 +// }
28 +//
29 +// ticker.Stop()
30 +// break
31 +// }
32 +//
33 +// if err != nil {
34 +// // Operation has failed.
35 +// }
36 +//
37 +// // Operation is successfull.
38 +//
39 +type Ticker struct {
40 + C <-chan time.Time
41 + c chan time.Time
42 + b BackOff
43 + stop chan struct{}
44 + stopOnce sync.Once
45 +}
46 +
47 +// NewTicker returns a new Ticker containing a channel that will send the time at times
48 +// specified by the BackOff argument. Ticker is guaranteed to tick at least once.
49 +// The channel is closed when Stop method is called or BackOff stops.
50 +func NewTicker(b BackOff) *Ticker {
51 + c := make(chan time.Time)
52 + t := &Ticker{
53 + C: c,
54 + c: c,
55 + b: b,
56 + stop: make(chan struct{}),
57 + }
58 + go t.run()
59 + runtime.SetFinalizer(t, (*Ticker).Stop)
60 + return t
61 +}
62 +
63 +// Stop turns off a ticker. After Stop, no more ticks will be sent.
64 +func (t *Ticker) Stop() {
65 + t.stopOnce.Do(func() { close(t.stop) })
66 +}
67 +
68 +func (t *Ticker) run() {
69 + c := t.c
70 + defer close(c)
71 + t.b.Reset()
72 +
73 + // Ticker is guaranteed to tick at least once.
74 + afterC := t.send(time.Now())
75 +
76 + for {
77 + if afterC == nil {
78 + return
79 + }
80 +
81 + select {
82 + case tick := <-afterC:
83 + afterC = t.send(tick)
84 + case <-t.stop:
85 + t.c = nil // Prevent future ticks from being sent to the channel.
86 + return
87 + }
88 + }
89 +}
90 +
91 +func (t *Ticker) send(tick time.Time) <-chan time.Time {
92 + select {
93 + case t.c <- tick:
94 + case <-t.stop:
95 + return nil
96 + }
97 +
98 + next := t.b.NextBackOff()
99 + if next == Stop {
100 + t.Stop()
101 + return nil
102 + }
103 +
104 + return time.After(next)
105 +}
Godeps/_workspace/src/github.com/cenkalti/backoff/ticker_test.go new
+45
@@ -0,0 +1,45 @@
1 +package backoff
2 +
3 +import (
4 + "errors"
5 + "log"
6 + "testing"
7 +)
8 +
9 +func TestTicker(t *testing.T) {
10 + const successOn = 3
11 + var i = 0
12 +
13 + // This function is successfull on "successOn" calls.
14 + f := func() error {
15 + i++
16 + log.Printf("function is called %d. time\n", i)
17 +
18 + if i == successOn {
19 + log.Println("OK")
20 + return nil
21 + }
22 +
23 + log.Println("error")
24 + return errors.New("error")
25 + }
26 +
27 + b := NewExponentialBackOff()
28 + ticker := NewTicker(b)
29 +
30 + var err error
31 + for _ = range ticker.C {
32 + if err = f(); err != nil {
33 + t.Log(err)
34 + continue
35 + }
36 +
37 + break
38 + }
39 + if err != nil {
40 + t.Errorf("unexpected error: %s", err.Error())
41 + }
42 + if i != successOn {
43 + t.Errorf("invalid number of retries: %d", i)
44 + }
45 +}
core/commands/mount_darwin.go
+1 -1
@@ -6,7 +6,7 @@ import (
6 "strings"
7 "syscall"
8
9 - fuseversion "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-fuse-version"
9 + fuseversion "github.com/jbenet/go-fuse-version"
10 )
11
12 func init() {
exchange/reprovide/reprovide.go
+14 -2
@@ -4,6 +4,7 @@ import (
4 "time"
5
6 context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
7 + backoff "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/cenkalti/backoff"
8
9 blocks "github.com/jbenet/go-ipfs/blocks/blockstore"
10 routing "github.com/jbenet/go-ipfs/routing"
@@ -50,9 +51,20 @@ func (rp *Reprovider) Reprovide(ctx context.Context) error {
51 return debugerror.Errorf("Failed to get key chan from blockstore: %s", err)
52 }
53 for k := range keychan {
53 - err := rp.rsys.Provide(ctx, k)
54 + op := func() error {
55 + err := rp.rsys.Provide(ctx, k)
56 + if err != nil {
57 + log.Warningf("Failed to provide key: %s", err)
58 + }
59 + return err
60 + }
61 +
62 + // TODO: this backoff library does not respect our context, we should
63 + // eventually work contexts into it. low priority.
64 + err := backoff.Retry(op, backoff.NewExponentialBackOff())
65 if err != nil {
55 - return debugerror.Errorf("Failed to provide key: %s, %s", k, err)
66 + log.Errorf("Providing failed after number of retries: %s", err)
67 + return err
68 }
69 }
70 return nil