plugin: create plugin API and loader, add ipld-git plugin
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Jul 4, 2017 at 11:48 UTC
7203c43b60ef63e4e1e5dea26a300c888a7d7ea9
18 files changed
+432
-22
Rules.mk
+3
@@ -56,6 +56,9 @@ include $(dir)/Rules.mk
56
dir := pin/internal/pb
57
include $(dir)/Rules.mk
58
59
+dir := plugin
60
+include $(dir)/Rules.mk
61
+
62
# -------------------- #
63
# universal rules #
64
# -------------------- #
cmd/ipfs/main.go
+7
@@ -11,6 +11,7 @@ import (
11
"net/url"
12
"os"
13
"os/signal"
14
+ "path/filepath"
15
"runtime/pprof"
16
"strings"
17
"sync"
@@ -22,6 +23,7 @@ import (
23
cmdsHttp "github.com/ipfs/go-ipfs/commands/http"
24
core "github.com/ipfs/go-ipfs/core"
25
coreCmds "github.com/ipfs/go-ipfs/core/commands"
26
+ "github.com/ipfs/go-ipfs/plugin/loader"
27
repo "github.com/ipfs/go-ipfs/repo"
28
config "github.com/ipfs/go-ipfs/repo/config"
29
fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
@@ -339,6 +341,11 @@ func callCommand(ctx context.Context, req cmds.Request, root *cmds.Command, cmd
341
} else {
342
log.Debug("executing command locally")
343
344
+ pluginpath := filepath.Join(req.InvocContext().ConfigRoot, "plugins")
345
+ if _, err := loader.LoadPlugins(pluginpath); err != nil {
346
+ return nil, err
347
+ }
348
+
349
err := req.SetRootContext(ctx)
350
if err != nil {
351
return nil, err
core/commands/dag/dag.go
+14
-22
@@ -7,6 +7,7 @@ import (
7
"strings"
8
9
cmds "github.com/ipfs/go-ipfs/commands"
10
+ coredag "github.com/ipfs/go-ipfs/core/coredag"
11
path "github.com/ipfs/go-ipfs/path"
12
pin "github.com/ipfs/go-ipfs/pin"
13
@@ -76,34 +77,25 @@ into an object of the specified format.
77
defer n.Blockstore.PinLock().Unlock()
78
}
79
79
- var c *cid.Cid
80
- switch ienc {
81
- case "json":
82
- nd, err := convertJsonToType(fi, format)
83
- if err != nil {
84
- res.SetError(err, cmds.ErrNormal)
85
- return
86
- }
80
+ nds, err := coredag.ParseInputs(ienc, format, fi)
81
+ if err != nil {
82
+ res.SetError(err, cmds.ErrNormal)
83
+ return
84
+ }
85
88
- c, err = n.DAG.Add(nd)
89
- if err != nil {
90
- res.SetError(err, cmds.ErrNormal)
91
- return
92
- }
93
- case "raw":
94
- nd, err := convertRawToType(fi, format)
86
+ var c *cid.Cid
87
+ b := n.DAG.Batch()
88
+ for _, nd := range nds {
89
+ cid, err := b.Add(nd)
90
if err != nil {
91
res.SetError(err, cmds.ErrNormal)
92
return
93
}
94
100
- c, err = n.DAG.Add(nd)
101
- if err != nil {
102
- res.SetError(err, cmds.ErrNormal)
103
- return
104
- }
105
- default:
106
- res.SetError(fmt.Errorf("unrecognized input encoding: %s", ienc), cmds.ErrNormal)
95
+ c = cid
96
+ }
97
+ if err := b.Commit(); err != nil {
98
+ res.SetError(err, cmds.ErrNormal)
99
return
100
}
101
core/coredag/dagtransl.go
new
+81
@@ -0,0 +1,81 @@
1
+package coredag
2
+
3
+import (
4
+ "fmt"
5
+ "io"
6
+ "io/ioutil"
7
+
8
+ node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
9
+ ipldcbor "gx/ipfs/QmemYymP73eVdTUUMZEiSpiHeZQKNJdT5dP2iuHssZh1sR/go-ipld-cbor"
10
+)
11
+
12
+type DagParser func(r io.Reader) ([]node.Node, error)
13
+
14
+type FormatParsers map[string]DagParser
15
+type InputEncParsers map[string]FormatParsers
16
+
17
+var DefaultInputEncParsers = InputEncParsers{
18
+ "json": DefaultJsonParsers,
19
+ "raw": DefaultRawParsers,
20
+}
21
+
22
+var DefaultJsonParsers = FormatParsers{
23
+ "cbor": CborJsonParser,
24
+ "dag-cbor": CborJsonParser,
25
+}
26
+
27
+var DefaultRawParsers = FormatParsers{
28
+ "cbor": CborRawParser,
29
+ "dag-cbor": CborRawParser,
30
+}
31
+
32
+func ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
33
+ return DefaultInputEncParsers.ParseInputs(ienc, format, r)
34
+}
35
+
36
+func (iep InputEncParsers) AddParser(ienv, format string, f DagParser) {
37
+ m, ok := iep[ienv]
38
+ if !ok {
39
+ m = make(FormatParsers)
40
+ iep[ienv] = m
41
+ }
42
+
43
+ m[format] = f
44
+}
45
+
46
+func (iep InputEncParsers) ParseInputs(ienc, format string, r io.Reader) ([]node.Node, error) {
47
+ pset, ok := iep[ienc]
48
+ if !ok {
49
+ return nil, fmt.Errorf("no input parser for %q", ienc)
50
+ }
51
+
52
+ parser, ok := pset[format]
53
+ if !ok {
54
+ return nil, fmt.Errorf("no parser for format %q using input type %q", format, ienc)
55
+ }
56
+
57
+ return parser(r)
58
+}
59
+
60
+func CborJsonParser(r io.Reader) ([]node.Node, error) {
61
+ nd, err := ipldcbor.FromJson(r)
62
+ if err != nil {
63
+ return nil, err
64
+ }
65
+
66
+ return []node.Node{nd}, nil
67
+}
68
+
69
+func CborRawParser(r io.Reader) ([]node.Node, error) {
70
+ data, err := ioutil.ReadAll(r)
71
+ if err != nil {
72
+ return nil, err
73
+ }
74
+
75
+ nd, err := ipldcbor.Decode(data)
76
+ if err != nil {
77
+ return nil, err
78
+ }
79
+
80
+ return []node.Node{nd}, nil
81
+}
package.json
+6
@@ -441,6 +441,12 @@
441
"hash": "QmPjTrrSfE6TzLv6ya6VWhGcCgPrUAdcgrDcQyRDX2VyW1",
442
"name": "go-libp2p-routing",
443
"version": "2.2.17"
444
+ },
445
+ {
446
+ "author": "whyrusleeping",
447
+ "hash": "Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex",
448
+ "name": "go-ipld-git",
449
+ "version": "0.1.3"
450
}
451
],
452
"gxVersion": "0.10.0",
plugin/Rules.mk
new
+9
@@ -0,0 +1,9 @@
1
+include mk/header.mk
2
+
3
+dir := $(d)/loader
4
+include $(dir)/Rules.mk
5
+
6
+dir := $(d)/plugins
7
+include $(dir)/Rules.mk
8
+
9
+include mk/footer.mk
plugin/ipld.go
new
+16
@@ -0,0 +1,16 @@
1
+package plugin
2
+
3
+import (
4
+ "github.com/ipfs/go-ipfs/core/coredag"
5
+
6
+ node "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
7
+)
8
+
9
+// PluginIPLD is an interface that can be implemented to add handlers for
10
+// for different IPLD formats
11
+type PluginIPLD interface {
12
+ Plugin
13
+
14
+ RegisterBlockDecoders(dec node.BlockDecoder) error
15
+ RegisterInputEncParsers(iec coredag.InputEncParsers) error
16
+}
plugin/loader/.gitignore
new
+1
@@ -0,0 +1 @@
1
+preload.go
plugin/loader/Rules.mk
new
+10
@@ -0,0 +1,10 @@
1
+include mk/header.mk
2
+
3
+$(d)/preload.go: d:=$(d)
4
+$(d)/preload.go: $(d)/preload_list
5
+ $(d)/preload.sh > $@
6
+ go fmt $@ >/dev/null
7
+
8
+DEPS_GO += $(d)/preload.go
9
+
10
+include mk/footer.mk
plugin/loader/initializer.go
new
+45
@@ -0,0 +1,45 @@
1
+package loader
2
+
3
+import (
4
+ "github.com/ipfs/go-ipfs/core/coredag"
5
+ "github.com/ipfs/go-ipfs/plugin"
6
+
7
+ format "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
8
+)
9
+
10
+func initalize(plugins []plugin.Plugin) error {
11
+ for _, p := range plugins {
12
+ err := p.Init()
13
+ if err != nil {
14
+ return err
15
+ }
16
+ }
17
+
18
+ return nil
19
+}
20
+
21
+func run(plugins []plugin.Plugin) error {
22
+ for _, pl := range plugins {
23
+ err := runIPLDPlugin(pl)
24
+ if err != nil {
25
+ return err
26
+ }
27
+ }
28
+ return nil
29
+}
30
+
31
+func runIPLDPlugin(pl plugin.Plugin) error {
32
+ ipldpl, ok := pl.(plugin.PluginIPLD)
33
+ if !ok {
34
+ return nil
35
+ }
36
+
37
+ var err error
38
+ err = ipldpl.RegisterBlockDecoders(format.DefaultBlockDecoder)
39
+ if err != nil {
40
+ return err
41
+ }
42
+
43
+ err = ipldpl.RegisterInputEncParsers(coredag.DefaultInputEncParsers)
44
+ return err
45
+}
plugin/loader/load.go
new
+49
@@ -0,0 +1,49 @@
1
+package loader
2
+
3
+import (
4
+ "fmt"
5
+
6
+ "github.com/ipfs/go-ipfs/plugin"
7
+
8
+ logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
9
+)
10
+
11
+var log = logging.Logger("plugin/loader")
12
+
13
+var loadPluginsFunc = func(string) ([]plugin.Plugin, error) {
14
+ return nil, nil
15
+}
16
+
17
+// LoadPlugins loads and initalizes plugins.
18
+func LoadPlugins(pluginDir string) ([]plugin.Plugin, error) {
19
+ plMap := make(map[string]plugin.Plugin)
20
+ for _, v := range preloadPlugins {
21
+ plMap[v.Name()] = v
22
+ }
23
+
24
+ newPls, err := loadPluginsFunc(pluginDir)
25
+ if err != nil {
26
+ return nil, err
27
+ }
28
+
29
+ for _, pl := range newPls {
30
+ if ppl, ok := plMap[pl.Name()]; ok {
31
+ // plugin is already preloaded
32
+ return nil, fmt.Errorf("plugin: %s, is duplicated in version: %s, while trying to load dynamically: %s", ppl.Name(), ppl.Version(), pl.Version())
33
+ }
34
+ plMap[pl.Name()] = pl
35
+ }
36
+
37
+ pls := make([]plugin.Plugin, 0, len(plMap))
38
+ for _, v := range plMap {
39
+ pls = append(pls, v)
40
+ }
41
+
42
+ err = initalize(pls)
43
+ if err != nil {
44
+ return nil, err
45
+ }
46
+
47
+ err = run(pls)
48
+ return nil, err
49
+}
plugin/loader/load_linux.go
new
+64
@@ -0,0 +1,64 @@
1
+package loader
2
+
3
+import (
4
+ "errors"
5
+ "fmt"
6
+ "os"
7
+ "path/filepath"
8
+ "plugin"
9
+
10
+ iplugin "github.com/ipfs/go-ipfs/plugin"
11
+)
12
+
13
+func init() {
14
+ loadPluginsFunc = linxuLoadFunc
15
+}
16
+
17
+func linxuLoadFunc(pluginDir string) ([]iplugin.Plugin, error) {
18
+ var plugins []iplugin.Plugin
19
+
20
+ filepath.Walk(pluginDir, func(fi string, info os.FileInfo, err error) error {
21
+ if err != nil {
22
+ return err
23
+ }
24
+ if info.IsDir() {
25
+ log.Warningf("found directory inside plugins directory: %s", fi)
26
+ return nil
27
+ }
28
+
29
+ if info.Mode().Perm()&0111 == 0 {
30
+ // file is not executable let's not load it
31
+ // this is to prevent loading plugins from for example non-executable
32
+ // mounts, some /tmp mounts are marked as such for security
33
+ log.Warningf("non-executable file in plugins directory: %s", fi)
34
+ return nil
35
+ }
36
+
37
+ if newPlugins, err := loadPlugin(fi); err == nil {
38
+ plugins = append(plugins, newPlugins...)
39
+ } else {
40
+ return fmt.Errorf("loading plugin %s: %s", fi, err)
41
+ }
42
+ return nil
43
+ })
44
+
45
+ return plugins, nil
46
+}
47
+
48
+func loadPlugin(fi string) ([]iplugin.Plugin, error) {
49
+ pl, err := plugin.Open(fi)
50
+ if err != nil {
51
+ return nil, err
52
+ }
53
+ pls, err := pl.Lookup("Plugins")
54
+ if err != nil {
55
+ return nil, err
56
+ }
57
+
58
+ typePls, ok := pls.([]iplugin.Plugin)
59
+ if !ok {
60
+ return nil, errors.New("filed 'Plugins' didn't contain correct type")
61
+ }
62
+
63
+ return typePls, nil
64
+}
plugin/loader/preload.sh
new
+31
@@ -0,0 +1,31 @@
1
+#!/bin/bash
2
+
3
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4
+
5
+to_preload() {
6
+ awk 'NF' "$DIR/preload_list" | sed '/^#/d'
7
+}
8
+
9
+cat <<EOL
10
+package loader
11
+
12
+import (
13
+ "github.com/ipfs/go-ipfs/plugin"
14
+EOL
15
+
16
+to_preload | while read -r name path num; do
17
+ echo "\tplugin$name \"$path\""
18
+done | sort -u
19
+
20
+cat <<EOL
21
+)
22
+
23
+var preloadPlugins = []plugin.Plugin{
24
+EOL
25
+
26
+to_preload | while read -r name path num; do
27
+ echo "\tplugin$name.Plugins[$num],"
28
+done
29
+
30
+
31
+echo "}"
plugin/loader/preload_list
new
+6
@@ -0,0 +1,6 @@
1
+# this file contains plugins to be preloaded
2
+# empty lines or starting with '#' are ignored
3
+#
4
+# name go-path number of the sub-plugin
5
+
6
+#ipldgit github.com/ipfs/go-ipfs/plugin/plugins/git 0
plugin/plugin.go
new
+12
@@ -0,0 +1,12 @@
1
+package plugin
2
+
3
+// Plugin is base interface for all kinds of go-ipfs plugins
4
+// It will be included in interfaces of different Plugins
5
+type Plugin interface {
6
+ // Name should return uniqe name of the plugin
7
+ Name() string
8
+ // Version returns current version of the plugin
9
+ Version() string
10
+ // Init is called once when the Plugin is being loaded
11
+ Init() error
12
+}
plugin/plugins/.gitignore
new
+1
@@ -0,0 +1 @@
1
+*.so
plugin/plugins/Rules.mk
new
+14
@@ -0,0 +1,14 @@
1
+include mk/header.mk
2
+
3
+$(d)_plugins:=$(d)/git
4
+$(d)_plugins_so:=$(addsuffix .so,$($(d)_plugins))
5
+
6
+$($(d)_plugins_so): $$(DEPS_GO) ALWAYS
7
+ go build -buildmode=plugin -i $(go-flags-with-tags) -o "$@" "$(call go-pkg-name,$(basename $@))"
8
+
9
+CLEAN += $($(d)_plugins_so)
10
+
11
+build_plugins: $($(d)_plugins_so)
12
+
13
+
14
+include mk/footer.mk
plugin/plugins/git/git.go
new
+63
@@ -0,0 +1,63 @@
1
+package git
2
+
3
+import (
4
+ "compress/zlib"
5
+ "io"
6
+
7
+ "github.com/ipfs/go-ipfs/core/coredag"
8
+ "github.com/ipfs/go-ipfs/plugin"
9
+
10
+ "gx/ipfs/QmTprEaAA2A9bst5XH7exuyi5KzNMK3SEDNN8rBDnKWcUS/go-cid"
11
+ "gx/ipfs/QmYNyRZJBUYPNrLszFmrBrPJbsBh2vMsefz5gnDpB5M1P6/go-ipld-format"
12
+ git "gx/ipfs/Qma7Kuwun7w8SZphjEPDVxvGfetBkqdNGmigDA13sJdLex/go-ipld-git"
13
+)
14
+
15
+var Plugins = []plugin.Plugin{
16
+ &GitPlugin{},
17
+}
18
+
19
+type GitPlugin struct{}
20
+
21
+var _ plugin.PluginIPLD = (*GitPlugin)(nil)
22
+
23
+func (*GitPlugin) Name() string {
24
+ return "ipld-git"
25
+}
26
+
27
+func (*GitPlugin) Version() string {
28
+ return "0.0.1"
29
+}
30
+
31
+func (*GitPlugin) Init() error {
32
+ return nil
33
+}
34
+
35
+func (*GitPlugin) RegisterBlockDecoders(dec format.BlockDecoder) error {
36
+ dec.Register(cid.GitRaw, git.DecodeBlock)
37
+ return nil
38
+}
39
+
40
+func (*GitPlugin) RegisterInputEncParsers(iec coredag.InputEncParsers) error {
41
+ iec.AddParser("raw", "git", parseRawGit)
42
+ iec.AddParser("zlib", "git", parseZlibGit)
43
+ return nil
44
+}
45
+
46
+func parseRawGit(r io.Reader) ([]format.Node, error) {
47
+ nd, err := git.ParseObject(r)
48
+ if err != nil {
49
+ return nil, err
50
+ }
51
+
52
+ return []format.Node{nd}, nil
53
+}
54
+
55
+func parseZlibGit(r io.Reader) ([]format.Node, error) {
56
+ rc, err := zlib.NewReader(r)
57
+ if err != nil {
58
+ return nil, err
59
+ }
60
+
61
+ defer rc.Close()
62
+ return parseRawGit(rc)
63
+}