S3 datastore support
To test it, set up an S3 bucket (in an AWS region that is not US Standard, for read-after-write consistency), run `ipfs init`, then edit `~/.ipfs/config` to say "Datastore": { "Type": "s3", "Region": "us-west-1", "Bucket": "mahbukkit", "ACL": "private" }, with the right values. Set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in the environment and you should be able to run `ipfs add` and `ipfs cat` and see the bucket be populated. No automated tests exist, unfortunately. S3 is thorny to simulate. License: MIT Signed-off-by: Tommi Virtanen <tv@eagain.net>
Tommi Virtanen committed
May 20, 2015 at 14:32 UTC
8f2d8204121a99344711636f7d9590b2d75acd2b
5 files changed
+79
-5
repo/config/datastore.go
+20
@@ -1,5 +1,9 @@
1
package config
2
3
+import (
4
+ "encoding/json"
5
+)
6
+
7
// DefaultDataStoreDirectory is the directory to store all the local IPFS data.
8
const DefaultDataStoreDirectory = "datastore"
9
@@ -10,6 +14,22 @@ type Datastore struct {
14
StorageMax string // in B, kB, kiB, MB, ...
15
StorageGCWatermark int64 // in percentage to multiply on StorageMax
16
GCPeriod string // in ns, us, ms, s, m, h
17
+
18
+ Params *json.RawMessage
19
+}
20
+
21
+func (d *Datastore) ParamData() []byte {
22
+ if d.Params == nil {
23
+ return nil
24
+ }
25
+
26
+ return []byte(*d.Params)
27
+}
28
+
29
+type S3Datastore struct {
30
+ Region string `json:"region"`
31
+ Bucket string `json:"bucket"`
32
+ ACL string `json:"acl"`
33
}
34
35
// DataStorePath returns the default data store path given a configuration root
repo/fsrepo/datastores.go
new
+38
@@ -0,0 +1,38 @@
1
+package fsrepo
2
+
3
+import (
4
+ "fmt"
5
+
6
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/aws"
7
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/crowdmob/goamz/s3"
8
+
9
+ repo "github.com/ipfs/go-ipfs/repo"
10
+ config "github.com/ipfs/go-ipfs/repo/config"
11
+ "github.com/ipfs/go-ipfs/thirdparty/s3-datastore"
12
+)
13
+
14
+func openS3Datastore(params config.S3Datastore) (repo.Datastore, error) {
15
+ // TODO support credentials files
16
+ auth, err := aws.EnvAuth()
17
+ if err != nil {
18
+ return nil, err
19
+ }
20
+
21
+ region := aws.GetRegion(params.Region)
22
+ if region.Name == "" {
23
+ return nil, fmt.Errorf("unknown AWS region: %q", params.Region)
24
+ }
25
+
26
+ if params.Bucket == "" {
27
+ return nil, fmt.Errorf("invalid S3 bucket: %q", params.Bucket)
28
+ }
29
+
30
+ client := s3.New(auth, region)
31
+ // There are too many gophermucking s3datastores in my
32
+ // gophermucking source.
33
+ return &s3datastore.S3Datastore{
34
+ Client: client,
35
+ Bucket: params.Bucket,
36
+ ACL: s3.ACL(params.ACL),
37
+ }, nil
38
+}
repo/fsrepo/fsrepo.go
+13
@@ -1,6 +1,7 @@
1
package fsrepo
2
3
import (
4
+ "encoding/json"
5
"errors"
6
"fmt"
7
"io"
@@ -331,6 +332,18 @@ func (r *FSRepo) openDatastore() error {
332
return err
333
}
334
r.ds = d
335
+ case "s3":
336
+ var dscfg config.S3Datastore
337
+ if err := json.Unmarshal(r.config.Datastore.ParamData(), &dscfg); err != nil {
338
+ return fmt.Errorf("datastore s3: %v", err)
339
+ }
340
+
341
+ ds, err := openS3Datastore(dscfg)
342
+ if err != nil {
343
+ return err
344
+ }
345
+
346
+ r.ds = ds
347
default:
348
return fmt.Errorf("unknown datastore type: %s", r.config.Datastore.Type)
349
}
repo/fsrepo/serialize/serialize_test.go
+4
-5
@@ -14,21 +14,20 @@ func TestConfig(t *testing.T) {
14
15
err := WriteConfigFile(filename, cfgWritten)
16
if err != nil {
17
- t.Error(err)
17
+ t.Fatal(err)
18
}
19
cfgRead, err := Load(filename)
20
if err != nil {
21
- t.Error(err)
22
- return
21
+ t.Fatal(err)
22
}
23
if cfgWritten.Identity.PeerID != cfgRead.Identity.PeerID {
25
- t.Fail()
24
+ t.Fatal()
25
}
26
st, err := os.Stat(filename)
27
if err != nil {
28
t.Fatalf("cannot stat config file: %v", err)
29
}
30
if g := st.Mode().Perm(); g&0117 != 0 {
32
- t.Errorf("config file should not be executable or accessible to world: %v", g)
31
+ t.Fatalf("config file should not be executable or accessible to world: %v", g)
32
}
33
}
thirdparty/s3-datastore/datastore.go
+4
@@ -67,4 +67,8 @@ func (ds *S3Datastore) Query(q query.Query) (query.Results, error) {
67
return nil, errors.New("TODO implement query for s3 datastore?")
68
}
69
70
+func (ds *S3Datastore) Close() error {
71
+ return nil
72
+}
73
+
74
func (ds *S3Datastore) IsThreadSafe() {}