add 1-to-2 migration program
Jeromy committed
Apr 8, 2015 at 16:18 UTC
2c79e5ddb51d2664a0a04deae8b97c04a6d76c6f
1 file changed
+366
repo/fsrepo/migrations/1-to-2/main.go
new
+366
@@ -0,0 +1,366 @@
1
+package main
2
+
3
+import (
4
+ "encoding/json"
5
+ "errors"
6
+ "fmt"
7
+ "io/ioutil"
8
+ "os"
9
+ "path"
10
+ "strings"
11
+
12
+ dstore "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
13
+ flatfs "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs"
14
+ leveldb "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb"
15
+ dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
16
+ migrate "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-migrate"
17
+ fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
18
+ mfsr "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
19
+
20
+ context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
21
+)
22
+
23
+var _ = context.Background
24
+
25
+const peerKeyName = "peer.key"
26
+
27
+type migration struct{}
28
+
29
+func (m migration) Versions() string {
30
+ return "1-to-2"
31
+}
32
+
33
+func (m migration) Reversible() bool {
34
+ return true
35
+}
36
+
37
+func (m migration) Apply(opts migrate.Options) error {
38
+ repo := mfsr.RepoPath(opts.Path)
39
+
40
+ if err := repo.CheckVersion("1"); err != nil {
41
+ return err
42
+ }
43
+
44
+ // 1) move key out of config, into its own path
45
+ err := moveKeyOutOfConfig(opts.Path)
46
+ if err != nil {
47
+ return err
48
+ }
49
+
50
+ // 2) Transfer blocks out of leveldb into flatDB
51
+ err = transferBlocksToFlatDB(opts.Path)
52
+ if err != nil {
53
+ return err
54
+ }
55
+
56
+ // 3) move ipfs path from .go-ipfs to .ipfs
57
+ newpath, err := moveIpfsDir(opts.Path)
58
+ if err != nil {
59
+ return err
60
+ }
61
+
62
+ // 4) Update version number
63
+ repo = mfsr.RepoPath(newpath)
64
+ err = repo.WriteVersion("2")
65
+ if err != nil {
66
+ return err
67
+ }
68
+
69
+ return nil
70
+}
71
+
72
+func (m migration) Revert(opts migrate.Options) error {
73
+ repo := mfsr.RepoPath(opts.Path)
74
+ if err := repo.CheckVersion("2"); err != nil {
75
+ return err
76
+ }
77
+
78
+ // 1) Move directory back to .go-ipfs
79
+ npath, err := reverseIpfsDir(opts.Path)
80
+ if err != nil {
81
+ return err
82
+ }
83
+
84
+ // 2) move blocks back from flatfs to leveldb
85
+ err = transferBlocksFromFlatDB(npath)
86
+ if err != nil {
87
+ return err
88
+ }
89
+
90
+ // 3) move key back into config
91
+ err = moveKeyIntoConfig(npath)
92
+ if err != nil {
93
+ return err
94
+ }
95
+
96
+ // 4) change version number back down
97
+ repo = mfsr.RepoPath(npath)
98
+ err = repo.WriteVersion("1")
99
+ if err != nil {
100
+ return err
101
+ }
102
+
103
+ return nil
104
+}
105
+
106
+func transferBlocksToFlatDB(repopath string) error {
107
+ r, err := fsrepo.Open(repopath)
108
+ if err != nil {
109
+ return err
110
+ }
111
+
112
+ blockspath := path.Join(repopath, "blocks")
113
+ err = os.Mkdir(blockspath, 0777)
114
+ if err != nil {
115
+ return err
116
+ }
117
+
118
+ fds, err := flatfs.New(blockspath, 4)
119
+ if err != nil {
120
+ return err
121
+ }
122
+
123
+ return transferBlocks(r.Datastore(), fds, "/b/", "")
124
+}
125
+
126
+func transferBlocksFromFlatDB(repopath string) error {
127
+
128
+ ldbpath := path.Join(repopath, "datastore")
129
+ blockspath := path.Join(repopath, "blocks")
130
+ fds, err := flatfs.New(blockspath, 4)
131
+ if err != nil {
132
+ return err
133
+ }
134
+
135
+ ldb, err := leveldb.NewDatastore(ldbpath, nil)
136
+ if err != nil {
137
+ return err
138
+ }
139
+
140
+ err = transferBlocks(fds, ldb, "", "/b/")
141
+ if err != nil {
142
+ return err
143
+ }
144
+
145
+ // Now remove the blocks directory
146
+ err = os.RemoveAll(blockspath)
147
+ if err != nil {
148
+ return err
149
+ }
150
+
151
+ return nil
152
+}
153
+
154
+func transferBlocks(from, to dstore.Datastore, fpref, tpref string) error {
155
+ q := dsq.Query{Prefix: fpref, KeysOnly: true}
156
+ res, err := from.Query(q)
157
+ if err != nil {
158
+ return err
159
+ }
160
+
161
+ fmt.Println("Starting query")
162
+ for result := range res.Next() {
163
+ nkey := fmt.Sprintf("%s%s", tpref, result.Key[len(fpref):])
164
+
165
+ fkey := dstore.NewKey(result.Key)
166
+ val, err := from.Get(fkey)
167
+ if err != nil {
168
+ return err
169
+ }
170
+
171
+ err = to.Put(dstore.NewKey(nkey), val)
172
+ if err != nil {
173
+ return err
174
+ }
175
+
176
+ err = from.Delete(fkey)
177
+ if err != nil {
178
+ return err
179
+ }
180
+ }
181
+ fmt.Println("Query done")
182
+
183
+ return nil
184
+}
185
+
186
+func moveKeyOutOfConfig(repopath string) error {
187
+ // Make keys directory
188
+ keypath := path.Join(repopath, "keys")
189
+ err := os.Mkdir(keypath, 0777)
190
+ if err != nil {
191
+ return err
192
+ }
193
+
194
+ // Grab the config
195
+ cfg, err := loadConfigJSON(repopath)
196
+ if err != nil {
197
+ return err
198
+ }
199
+
200
+ // get the private key from it
201
+ privKey, err := getPrivateKeyFromConfig(cfg)
202
+ if err != nil {
203
+ return err
204
+ }
205
+
206
+ keyfilepath := path.Join(keypath, peerKeyName)
207
+ fi, err := os.OpenFile(keyfilepath, os.O_CREATE|os.O_WRONLY, 0600)
208
+ if err != nil {
209
+ return err
210
+ }
211
+
212
+ // Write our b64-protobuf encoded key
213
+ _, err = fi.WriteString(privKey)
214
+ if err != nil {
215
+ return err
216
+ }
217
+
218
+ err = fi.Close()
219
+ if err != nil {
220
+ return err
221
+ }
222
+
223
+ // Now that the key is safely in its own file, remove it from the config
224
+ err = clearPrivateKeyFromConfig(cfg)
225
+ if err != nil {
226
+ return err
227
+ }
228
+
229
+ err = saveConfigJSON(repopath, cfg)
230
+ if err != nil {
231
+ return err
232
+ }
233
+
234
+ return nil
235
+}
236
+
237
+// Part of the 2-to-1 revert process
238
+func moveKeyIntoConfig(repopath string) error {
239
+ // Make keys directory
240
+ keypath := path.Join(repopath, "keys")
241
+
242
+ // Grab the config
243
+ cfg, err := loadConfigJSON(repopath)
244
+ if err != nil {
245
+ return err
246
+ }
247
+
248
+ keyfilepath := path.Join(keypath, peerKeyName)
249
+ pkey, err := ioutil.ReadFile(keyfilepath)
250
+ if err != nil {
251
+ return err
252
+ }
253
+
254
+ id, ok := cfg["Identity"]
255
+ if !ok {
256
+ return errors.New("expected to find an identity object in config")
257
+ }
258
+ identity, ok := id.(map[string]interface{})
259
+ if !ok {
260
+ return errors.New("expected Identity in config to be an object")
261
+ }
262
+ identity["PrivKey"] = string(pkey)
263
+
264
+ err = saveConfigJSON(repopath, cfg)
265
+ if err != nil {
266
+ return err
267
+ }
268
+
269
+ // Now that the key is safely in the config, delete the file
270
+ err = os.RemoveAll(keypath)
271
+ if err != nil {
272
+ return err
273
+ }
274
+
275
+ return nil
276
+}
277
+
278
+func moveIpfsDir(curpath string) (string, error) {
279
+ newpath := strings.Replace(curpath, ".go-ipfs", ".ipfs", 1)
280
+ return newpath, os.Rename(curpath, newpath)
281
+}
282
+
283
+func reverseIpfsDir(curpath string) (string, error) {
284
+ newpath := strings.Replace(curpath, ".ipfs", ".go-ipfs", 1)
285
+ return newpath, os.Rename(curpath, newpath)
286
+}
287
+
288
+func loadConfigJSON(repoPath string) (map[string]interface{}, error) {
289
+ cfgPath := path.Join(repoPath, "config")
290
+ fi, err := os.Open(cfgPath)
291
+ if err != nil {
292
+ return nil, err
293
+ }
294
+
295
+ var out map[string]interface{}
296
+ err = json.NewDecoder(fi).Decode(&out)
297
+ if err != nil {
298
+ return nil, err
299
+ }
300
+
301
+ return out, nil
302
+}
303
+
304
+func saveConfigJSON(repoPath string, cfg map[string]interface{}) error {
305
+ cfgPath := path.Join(repoPath, "config")
306
+ fi, err := os.Create(cfgPath)
307
+ if err != nil {
308
+ return err
309
+ }
310
+
311
+ out, err := json.MarshalIndent(cfg, "", "\t")
312
+ if err != nil {
313
+ return err
314
+ }
315
+
316
+ _, err = fi.Write(out)
317
+ if err != nil {
318
+ return err
319
+ }
320
+
321
+ return nil
322
+}
323
+
324
+func getPrivateKeyFromConfig(cfg map[string]interface{}) (string, error) {
325
+ ident, ok := cfg["Identity"]
326
+ if !ok {
327
+ return "", errors.New("no identity found in config")
328
+ }
329
+
330
+ identMap, ok := ident.(map[string]interface{})
331
+ if !ok {
332
+ return "", errors.New("expected Identity to be object (map)")
333
+ }
334
+
335
+ privkey, ok := identMap["PrivKey"]
336
+ if !ok {
337
+ return "", errors.New("no PrivKey field found in Identity")
338
+ }
339
+
340
+ privkeyStr, ok := privkey.(string)
341
+ if !ok {
342
+ return "", errors.New("expected PrivKey to be a string")
343
+ }
344
+
345
+ return privkeyStr, nil
346
+}
347
+
348
+func clearPrivateKeyFromConfig(cfg map[string]interface{}) error {
349
+ ident, ok := cfg["Identity"]
350
+ if !ok {
351
+ return errors.New("no identity found in config")
352
+ }
353
+
354
+ identMap, ok := ident.(map[string]interface{})
355
+ if !ok {
356
+ return errors.New("expected Identity to be object (map)")
357
+ }
358
+
359
+ delete(identMap, "PrivKey")
360
+ return nil
361
+}
362
+
363
+func main() {
364
+ m := migration{}
365
+ migrate.Main(&m)
366
+}