fix(ipfswatch): loading datastore plugins (#11078)
* ipfswatch: fix loading datastore plugins * test: add CLI tests for ipfswatch --------- Co-authored-by: Marcin Rataj <lidel@lidel.org>
Andrew Gillis committed
Dec 9, 2025 at 17:50 UTC
ab44726177ed1f0dc2fcfc7c5752989fe7ba01be
2 files changed
+189
cmd/ipfswatch/main.go
+27
@@ -9,6 +9,7 @@ import (
9
"os"
10
"os/signal"
11
"path/filepath"
12
+ "slices"
13
"syscall"
14
15
commands "github.com/ipfs/kubo/commands"
@@ -17,6 +18,11 @@ import (
18
coreapi "github.com/ipfs/kubo/core/coreapi"
19
corehttp "github.com/ipfs/kubo/core/corehttp"
20
"github.com/ipfs/kubo/misc/fsutil"
21
+ "github.com/ipfs/kubo/plugin"
22
+ pluginbadgerds "github.com/ipfs/kubo/plugin/plugins/badgerds"
23
+ pluginflatfs "github.com/ipfs/kubo/plugin/plugins/flatfs"
24
+ pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds"
25
+ pluginpebbleds "github.com/ipfs/kubo/plugin/plugins/pebbleds"
26
fsrepo "github.com/ipfs/kubo/repo/fsrepo"
27
28
fsnotify "github.com/fsnotify/fsnotify"
@@ -60,6 +66,18 @@ func main() {
66
}
67
}
68
69
+func loadDatastorePlugins(plugins []plugin.Plugin) error {
70
+ for _, pl := range plugins {
71
+ if pl, ok := pl.(plugin.PluginDatastore); ok {
72
+ err := fsrepo.AddDatastoreConfigHandler(pl.DatastoreTypeName(), pl.DatastoreConfigParser())
73
+ if err != nil {
74
+ return err
75
+ }
76
+ }
77
+ }
78
+ return nil
79
+}
80
+
81
func run(ipfsPath, watchPath string) error {
82
log.Printf("running IPFSWatch on '%s' using repo at '%s'...", watchPath, ipfsPath)
83
@@ -77,6 +95,15 @@ func run(ipfsPath, watchPath string) error {
95
return err
96
}
97
98
+ if err = loadDatastorePlugins(slices.Concat(
99
+ pluginbadgerds.Plugins,
100
+ pluginflatfs.Plugins,
101
+ pluginlevelds.Plugins,
102
+ pluginpebbleds.Plugins,
103
+ )); err != nil {
104
+ return err
105
+ }
106
+
107
r, err := fsrepo.Open(ipfsPath)
108
if err != nil {
109
// TODO handle case: daemon running
test/cli/ipfswatch_test.go
new
+162
@@ -0,0 +1,162 @@
1
+//go:build !plan9
2
+
3
+package cli
4
+
5
+import (
6
+ "fmt"
7
+ "os"
8
+ "os/exec"
9
+ "path/filepath"
10
+ "regexp"
11
+ "testing"
12
+ "time"
13
+
14
+ "github.com/ipfs/kubo/config"
15
+ "github.com/ipfs/kubo/test/cli/harness"
16
+ "github.com/stretchr/testify/require"
17
+)
18
+
19
+func TestIPFSWatch(t *testing.T) {
20
+ t.Parallel()
21
+
22
+ // Build ipfswatch binary once before running parallel subtests.
23
+ // This avoids race conditions and duplicate builds.
24
+ h := harness.NewT(t)
25
+ repoRoot := filepath.Dir(filepath.Dir(filepath.Dir(h.IPFSBin)))
26
+ ipfswatchBin := filepath.Join(repoRoot, "cmd", "ipfswatch", "ipfswatch")
27
+
28
+ if _, err := os.Stat(ipfswatchBin); os.IsNotExist(err) {
29
+ // -C changes to repo root so go.mod is found
30
+ cmd := exec.Command("go", "build", "-C", repoRoot, "-o", ipfswatchBin, "./cmd/ipfswatch")
31
+ out, err := cmd.CombinedOutput()
32
+ require.NoError(t, err, "failed to build ipfswatch: %s", string(out))
33
+ }
34
+
35
+ t.Run("ipfswatch adds watched files to IPFS", func(t *testing.T) {
36
+ t.Parallel()
37
+ h := harness.NewT(t)
38
+ node := h.NewNode().Init()
39
+
40
+ // Create a temp directory to watch
41
+ watchDir := filepath.Join(h.Dir, "watch")
42
+ err := os.MkdirAll(watchDir, 0o755)
43
+ require.NoError(t, err)
44
+
45
+ // Start ipfswatch in background
46
+ result := node.Runner.Run(harness.RunRequest{
47
+ Path: ipfswatchBin,
48
+ Args: []string{"--repo", node.Dir, "--path", watchDir},
49
+ RunFunc: harness.RunFuncStart,
50
+ })
51
+ require.NoError(t, result.Err, "ipfswatch should start without error")
52
+ defer func() {
53
+ if result.Cmd.Process != nil {
54
+ _ = result.Cmd.Process.Kill()
55
+ _, _ = result.Cmd.Process.Wait()
56
+ }
57
+ }()
58
+
59
+ // Wait for ipfswatch to initialize
60
+ time.Sleep(2 * time.Second)
61
+
62
+ // Check for startup errors
63
+ stderrStr := result.Stderr.String()
64
+ require.NotContains(t, stderrStr, "unknown datastore type", "ipfswatch should recognize datastore plugins")
65
+
66
+ // Create a test file with unique content based on timestamp
67
+ testContent := fmt.Sprintf("ipfswatch test content generated at %s", time.Now().Format(time.RFC3339Nano))
68
+ testFile := filepath.Join(watchDir, "test.txt")
69
+ err = os.WriteFile(testFile, []byte(testContent), 0o644)
70
+ require.NoError(t, err)
71
+
72
+ // Wait for ipfswatch to process the file and extract CID from log
73
+ // Log format: "added %s... key: %s"
74
+ cidPattern := regexp.MustCompile(`added .*/test\.txt\.\.\. key: (\S+)`)
75
+ var cid string
76
+ deadline := time.Now().Add(10 * time.Second)
77
+ for time.Now().Before(deadline) {
78
+ stderrStr = result.Stderr.String()
79
+ if matches := cidPattern.FindStringSubmatch(stderrStr); len(matches) > 1 {
80
+ cid = matches[1]
81
+ break
82
+ }
83
+ time.Sleep(100 * time.Millisecond)
84
+ }
85
+ require.NotEmpty(t, cid, "ipfswatch should have added test.txt and logged the CID, got stderr: %s", stderrStr)
86
+
87
+ // Kill ipfswatch to release the repo lock
88
+ if result.Cmd.Process != nil {
89
+ _ = result.Cmd.Process.Kill()
90
+ _, _ = result.Cmd.Process.Wait()
91
+ }
92
+
93
+ // Verify the content matches by reading it back via ipfs cat
94
+ catRes := node.RunIPFS("cat", "--offline", cid)
95
+ require.Equal(t, 0, catRes.Cmd.ProcessState.ExitCode(),
96
+ "ipfs cat should succeed, cid=%s, stderr: %s", cid, catRes.Stderr.String())
97
+ require.Equal(t, testContent, catRes.Stdout.String(),
98
+ "content read from IPFS should match what was written")
99
+ })
100
+
101
+ t.Run("ipfswatch loads datastore plugins for pebbleds", func(t *testing.T) {
102
+ t.Parallel()
103
+ h := harness.NewT(t)
104
+ node := h.NewNode().Init()
105
+
106
+ // Configure pebbleds as the datastore
107
+ node.UpdateConfig(func(cfg *config.Config) {
108
+ cfg.Datastore.Spec = map[string]interface{}{
109
+ "type": "mount",
110
+ "mounts": []interface{}{
111
+ map[string]interface{}{
112
+ "mountpoint": "/blocks",
113
+ "path": "blocks",
114
+ "prefix": "flatfs.datastore",
115
+ "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
116
+ "sync": true,
117
+ "type": "flatfs",
118
+ },
119
+ map[string]interface{}{
120
+ "mountpoint": "/",
121
+ "path": "datastore",
122
+ "prefix": "pebble.datastore",
123
+ "type": "pebbleds",
124
+ },
125
+ },
126
+ }
127
+ })
128
+
129
+ // Re-initialize datastore directory for pebbleds
130
+ // (the repo was initialized with levelds, need to remove it)
131
+ dsPath := filepath.Join(node.Dir, "datastore")
132
+ err := os.RemoveAll(dsPath)
133
+ require.NoError(t, err)
134
+ err = os.MkdirAll(dsPath, 0o755)
135
+ require.NoError(t, err)
136
+
137
+ // Create a temp directory to watch
138
+ watchDir := filepath.Join(h.Dir, "watch")
139
+ err = os.MkdirAll(watchDir, 0o755)
140
+ require.NoError(t, err)
141
+
142
+ // Start ipfswatch in background
143
+ result := node.Runner.Run(harness.RunRequest{
144
+ Path: ipfswatchBin,
145
+ Args: []string{"--repo", node.Dir, "--path", watchDir},
146
+ RunFunc: harness.RunFuncStart,
147
+ })
148
+ require.NoError(t, result.Err, "ipfswatch should start without error")
149
+ defer func() {
150
+ if result.Cmd.Process != nil {
151
+ _ = result.Cmd.Process.Kill()
152
+ _, _ = result.Cmd.Process.Wait()
153
+ }
154
+ }()
155
+
156
+ // Wait for ipfswatch to initialize and check for errors
157
+ time.Sleep(3 * time.Second)
158
+
159
+ stderrStr := result.Stderr.String()
160
+ require.NotContains(t, stderrStr, "unknown datastore type", "ipfswatch should recognize pebbleds datastore plugin")
161
+ })
162
+}