go-ipfs-config: feat: mark badger as stable
We're still not ready to enable it by default but at least mark it as stable.
Steven Allen committed
Mar 25, 2020 at 15:35 UTC
cc723f94b8b1fd35b8c1914d25b86fde7f892319
2 files changed
+82
-60
config/init.go
+39
-22
@@ -118,29 +118,46 @@ func DefaultDatastoreConfig() Datastore {
118
StorageGCWatermark: 90, // 90%
119
GCPeriod: "1h",
120
BloomFilterSize: 0,
121
- Spec: map[string]interface{}{
122
- "type": "mount",
123
- "mounts": []interface{}{
124
- map[string]interface{}{
125
- "mountpoint": "/blocks",
126
- "type": "measure",
127
- "prefix": "flatfs.datastore",
128
- "child": map[string]interface{}{
129
- "type": "flatfs",
130
- "path": "blocks",
131
- "sync": true,
132
- "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
133
- },
121
+ Spec: flatfsSpec(),
122
+ }
123
+}
124
+
125
+func badgerSpec() map[string]interface{} {
126
+ return map[string]interface{}{
127
+ "type": "measure",
128
+ "prefix": "badger.datastore",
129
+ "child": map[string]interface{}{
130
+ "type": "badgerds",
131
+ "path": "badgerds",
132
+ "syncWrites": false,
133
+ "truncate": true,
134
+ },
135
+ }
136
+}
137
+
138
+func flatfsSpec() map[string]interface{} {
139
+ return map[string]interface{}{
140
+ "type": "mount",
141
+ "mounts": []interface{}{
142
+ map[string]interface{}{
143
+ "mountpoint": "/blocks",
144
+ "type": "measure",
145
+ "prefix": "flatfs.datastore",
146
+ "child": map[string]interface{}{
147
+ "type": "flatfs",
148
+ "path": "blocks",
149
+ "sync": true,
150
+ "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
151
},
135
- map[string]interface{}{
136
- "mountpoint": "/",
137
- "type": "measure",
138
- "prefix": "leveldb.datastore",
139
- "child": map[string]interface{}{
140
- "type": "levelds",
141
- "path": "datastore",
142
- "compression": "none",
143
- },
152
+ },
153
+ map[string]interface{}{
154
+ "mountpoint": "/",
155
+ "type": "measure",
156
+ "prefix": "leveldb.datastore",
157
+ "child": map[string]interface{}{
158
+ "type": "levelds",
159
+ "path": "datastore",
160
+ "compression": "none",
161
},
162
},
163
},
config/profile.go
+43
-38
@@ -11,11 +11,14 @@ type Transformer func(c *Config) error
11
12
// Profile contains the profile transformer the description of the profile
13
type Profile struct {
14
- // Description briefly describes the functionality of the profile
14
+ // Description briefly describes the functionality of the profile.
15
Description string
16
17
- // Transform takes ipfs configuration and applies the profile to it
17
+ // Transform takes ipfs configuration and applies the profile to it.
18
Transform Transformer
19
+
20
+ // InitOnly specifies that this profile can only be applied on init.
21
+ InitOnly bool
22
}
23
24
// defaultServerFilters has is a list of IPv4 and IPv6 prefixes that are private, local only, or unrouteable.
@@ -107,51 +110,49 @@ Inverse profile of the test profile.`,
110
return nil
111
},
112
},
110
- "badgerds": {
111
- Description: `Replaces default datastore configuration with experimental
112
-badger datastore.
113
-
114
-If you apply this profile after ipfs init, you will need
115
-to convert your datastore to the new configuration.
116
-You can do this using ipfs-ds-convert.
117
-
118
-For more on ipfs-ds-convert see
119
-$ ipfs-ds-convert --help
120
-and
121
-$ ipfs-ds-convert convert --help
122
-
123
-WARNING: badger datastore is experimental.
124
-Make sure to backup your data frequently.`,
113
+ "flatfs": {
114
+ Description: `Configures the node to use the flatfs datastore.
115
+
116
+This is the most battle-tested and reliable datastore, but it's significantly
117
+slower than the badger datastore. You should use this datastore if:
118
+
119
+* You need a very simple and very reliable datastore you and trust your
120
+ filesystem. This datastore stores each block as a separate file in the
121
+ underlying filesystem so it's unlikely to loose data unless there's an issue
122
+ with the underlying file system.
123
+* You need to run garbage collection on a small (<= 10GiB) datastore. The
124
+ default datastore, badger, can leave several gigabytes of data behind when
125
+ garbage collecting.
126
+* You're concerned about memory usage. In its default configuration, badger can
127
+ use up to several gigabytes of memory.
128
+
129
+This profile may only be applied when first initializing the node.
130
+`,
131
132
+ InitOnly: true,
133
Transform: func(c *Config) error {
127
- c.Datastore.Spec = map[string]interface{}{
128
- "type": "measure",
129
- "prefix": "badger.datastore",
130
- "child": map[string]interface{}{
131
- "type": "badgerds",
132
- "path": "badgerds",
133
- "syncWrites": false,
134
- "truncate": true,
135
- },
136
- }
134
+ c.Datastore.Spec = flatfsSpec()
135
return nil
136
},
137
},
140
- "default-datastore": {
141
- Description: `Restores default datastore configuration.
138
+ "badgerds": {
139
+ Description: `Configures the node to use the badger datastore.
140
143
-If you apply this profile after ipfs init, you will need
144
-to convert your datastore to the new configuration.
145
-You can do this using ipfs-ds-convert.
141
+This is the fastest datastore. Use this datastore if performance, especially
142
+when adding many gigabytes of files, is critical. However:
143
147
-For more on ipfs-ds-convert see
148
-$ ipfs-ds-convert --help
149
-and
150
-$ ipfs-ds-convert convert --help
151
-`,
144
+* This datastore will not properly reclaim space when your datastore is
145
+ smaller than several gigabytes. If you run IPFS with '--enable-gc' (you have
146
+ enabled block-level garbage collection), you plan on storing very little data in
147
+ your IPFS node, and disk usage is more critical than performance, consider using
148
+ flatfs.
149
+* This datastore uses up to several gigabytes of memory.
150
151
+This profile may only be applied when first initializing the node.`,
152
+
153
+ InitOnly: true,
154
Transform: func(c *Config) error {
154
- c.Datastore.Spec = DefaultDatastoreConfig().Spec
155
+ c.Datastore.Spec = badgerSpec()
156
return nil
157
},
158
},
@@ -187,6 +188,10 @@ fetching may be degraded.
188
},
189
}
190
191
+func init() {
192
+ Profiles["default-datatore"] = Profiles["badgerds"]
193
+}
194
+
195
func getAvailablePort() (port int, err error) {
196
ln, err := net.Listen("tcp", "[::]:0")
197
if err != nil {