@cryptotaxi247 / netdata-1 / commits / bf9c2c0e4

go.d filecheck update to create a chart per instance (#17624)

Ilya Mashchenko committed May 9, 2024 at 14:25 UTC bf9c2c0e4ed49eb1c30b4dba5478509fba91e120
11 files changed +638 -514
src/go/collectors/go.d.plugin/modules/filecheck/cache.go new
+28
@@ -0,0 +1,28 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package filecheck
4 +
5 +func newSeenItems() *seenItems {
6 + return &seenItems{
7 + items: make(map[string]*seenItem),
8 + }
9 +}
10 +
11 +type (
12 + seenItems struct {
13 + items map[string]*seenItem
14 + }
15 + seenItem struct {
16 + hasExistenceCharts bool
17 + hasOtherCharts bool
18 + }
19 +)
20 +
21 +func (c *seenItems) getp(path string) *seenItem {
22 + item, ok := c.items[path]
23 + if !ok {
24 + item = &seenItem{}
25 + c.items[path] = item
26 + }
27 + return item
28 +}
src/go/collectors/go.d.plugin/modules/filecheck/charts.go
+249 -62
@@ -2,78 +2,265 @@
2
3 package filecheck
4
5 -import "github.com/netdata/netdata/go/go.d.plugin/agent/module"
5 +import (
6 + "fmt"
7 + "strings"
8 +
9 + "github.com/netdata/netdata/go/go.d.plugin/agent/module"
10 +)
11 +
12 +const (
13 + prioFileExistenceStatus = module.Priority + iota
14 + prioFileModificationTimeAgo
15 + prioFileSize
16 +
17 + prioDirExistenceStatus
18 + prioDirModificationTimeAgo
19 + prioDirSize
20 + prioDirFilesCount
21 +)
22
23 var (
8 - fileCharts = module.Charts{
9 - fileExistenceChart.Copy(),
10 - fileModTimeAgoChart.Copy(),
11 - fileSizeChart.Copy(),
12 - }
13 -
14 - fileExistenceChart = module.Chart{
15 - ID: "file_existence",
16 - Title: "File Existence (0: not exists, 1: exists)",
17 - Units: "boolean",
18 - Fam: "files",
19 - Ctx: "filecheck.file_existence",
20 - Vars: module.Vars{
21 - {ID: "num_of_files"},
24 + fileExistenceStatusChartTmpl = module.Chart{
25 + ID: "file_%s_existence_status",
26 + Title: "File existence",
27 + Units: "status",
28 + Fam: "file existence",
29 + Ctx: "filecheck.file_existence_status",
30 + Priority: prioFileExistenceStatus,
31 + Dims: module.Dims{
32 + {ID: "file_%s_existence_status_exist", Name: "exist"},
33 + {ID: "file_%s_existence_status_not_exist", Name: "not_exist"},
34 },
35 }
24 - fileModTimeAgoChart = module.Chart{
25 - ID: "file_mtime_ago",
26 - Title: "File Time Since the Last Modification",
27 - Units: "seconds",
28 - Fam: "files",
29 - Ctx: "filecheck.file_mtime_ago",
36 +
37 + fileModificationTimeAgoChartTmpl = module.Chart{
38 + ID: "file_%s_modification_time_ago",
39 + Title: "File time since the last modification",
40 + Units: "seconds",
41 + Fam: "file mtime",
42 + Ctx: "filecheck.file_modification_time_ago",
43 + Priority: prioFileModificationTimeAgo,
44 + Dims: module.Dims{
45 + {ID: "file_%s_mtime_ago", Name: "mtime_ago"},
46 + },
47 }
31 - fileSizeChart = module.Chart{
32 - ID: "file_size",
33 - Title: "File Size",
34 - Units: "bytes",
35 - Fam: "files",
36 - Ctx: "filecheck.file_size",
48 + fileSizeChartTmpl = module.Chart{
49 + ID: "file_%s_size",
50 + Title: "File size",
51 + Units: "bytes",
52 + Fam: "file size",
53 + Ctx: "filecheck.file_size_bytes",
54 + Priority: prioFileSize,
55 + Dims: module.Dims{
56 + {ID: "file_%s_size_bytes", Name: "size"},
57 + },
58 }
59 )
60
61 var (
41 - dirCharts = module.Charts{
42 - dirExistenceChart.Copy(),
43 - dirModTimeChart.Copy(),
44 - dirNumOfFilesChart.Copy(),
45 - dirSizeChart.Copy(),
46 - }
47 -
48 - dirExistenceChart = module.Chart{
49 - ID: "dir_existence",
50 - Title: "Dir Existence (0: not exists, 1: exists)",
51 - Units: "boolean",
52 - Fam: "dirs",
53 - Ctx: "filecheck.dir_existence",
54 - Vars: module.Vars{
55 - {ID: "num_of_dirs"},
62 + dirExistenceStatusChartTmpl = module.Chart{
63 + ID: "dir_%s_existence_status",
64 + Title: "Directory existence",
65 + Units: "status",
66 + Fam: "dir existence",
67 + Ctx: "filecheck.dir_existence_status",
68 + Priority: prioDirExistenceStatus,
69 + Dims: module.Dims{
70 + {ID: "dir_%s_existence_status_exist", Name: "exist"},
71 + {ID: "dir_%s_existence_status_not_exist", Name: "not_exist"},
72 + },
73 + }
74 +
75 + dirModificationTimeAgoChartTmpl = module.Chart{
76 + ID: "dir_%s_modification_time_ago",
77 + Title: "Directory time since the last modification",
78 + Units: "seconds",
79 + Fam: "dir mtime",
80 + Ctx: "filecheck.dir_modification_time_ago",
81 + Priority: prioDirModificationTimeAgo,
82 + Dims: module.Dims{
83 + {ID: "dir_%s_mtime_ago", Name: "mtime_ago"},
84 },
85 }
58 - dirModTimeChart = module.Chart{
59 - ID: "dir_mtime_ago",
60 - Title: "Dir Time Since the Last Modification",
61 - Units: "seconds",
62 - Fam: "dirs",
63 - Ctx: "filecheck.dir_mtime_ago",
64 - }
65 - dirNumOfFilesChart = module.Chart{
66 - ID: "dir_num_of_files",
67 - Title: "Dir Number of Files",
68 - Units: "files",
69 - Fam: "dirs",
70 - Ctx: "filecheck.dir_num_of_files",
71 - }
72 - dirSizeChart = module.Chart{
73 - ID: "dir_size",
74 - Title: "Dir Size",
75 - Units: "bytes",
76 - Fam: "dirs",
77 - Ctx: "filecheck.dir_size",
86 + dirSizeChartTmpl = module.Chart{
87 + ID: "dir_%s_size",
88 + Title: "Directory size",
89 + Units: "bytes",
90 + Fam: "dir size",
91 + Ctx: "filecheck.dir_size_bytes",
92 + Priority: prioDirSize,
93 + Dims: module.Dims{
94 + {ID: "dir_%s_size_bytes", Name: "size"},
95 + },
96 + }
97 + dirFilesCountChartTmpl = module.Chart{
98 + ID: "dir_%s_files_count",
99 + Title: "Directory files count",
100 + Units: "files",
101 + Fam: "dir files",
102 + Ctx: "filecheck.dir_files_count",
103 + Priority: prioDirFilesCount,
104 + Dims: module.Dims{
105 + {ID: "dir_%s_files_count", Name: "files"},
106 + },
107 }
108 )
109 +
110 +func (f *Filecheck) updateFileCharts(infos []*statInfo) {
111 + seen := make(map[string]bool)
112 +
113 + for _, info := range infos {
114 + seen[info.path] = true
115 +
116 + sf := f.seenFiles.getp(info.path)
117 +
118 + if !sf.hasExistenceCharts {
119 + sf.hasExistenceCharts = true
120 + f.addFileCharts(info.path,
121 + fileExistenceStatusChartTmpl.Copy(),
122 + )
123 + }
124 +
125 + if !sf.hasOtherCharts && info.fi != nil {
126 + sf.hasOtherCharts = true
127 + f.addFileCharts(info.path,
128 + fileModificationTimeAgoChartTmpl.Copy(),
129 + fileSizeChartTmpl.Copy(),
130 + )
131 +
132 + } else if sf.hasOtherCharts && info.fi == nil {
133 + sf.hasOtherCharts = false
134 + f.removeFileOtherCharts(info.path)
135 + }
136 + }
137 +
138 + for path := range f.seenFiles.items {
139 + if !seen[path] {
140 + delete(f.seenFiles.items, path)
141 + f.removeFileAllCharts(path)
142 + }
143 + }
144 +}
145 +
146 +func (f *Filecheck) updateDirCharts(infos []*statInfo) {
147 + seen := make(map[string]bool)
148 +
149 + for _, info := range infos {
150 + seen[info.path] = true
151 +
152 + sd := f.seenDirs.getp(info.path)
153 +
154 + if !sd.hasExistenceCharts {
155 + sd.hasExistenceCharts = true
156 + f.addFileCharts(info.path,
157 + dirExistenceStatusChartTmpl.Copy(),
158 + )
159 + }
160 +
161 + if !sd.hasOtherCharts && info.fi != nil {
162 + sd.hasOtherCharts = true
163 + f.addDirCharts(info.path,
164 + dirModificationTimeAgoChartTmpl.Copy(),
165 + dirFilesCountChartTmpl.Copy(),
166 + )
167 + if f.Dirs.CollectDirSize {
168 + f.addDirCharts(info.path,
169 + dirSizeChartTmpl.Copy(),
170 + )
171 + }
172 +
173 + } else if sd.hasOtherCharts && info.fi == nil {
174 + sd.hasOtherCharts = false
175 + f.removeDirOtherCharts(info.path)
176 + }
177 + }
178 +
179 + for path := range f.seenDirs.items {
180 + if !seen[path] {
181 + delete(f.seenDirs.items, path)
182 + f.removeDirAllCharts(path)
183 + }
184 + }
185 +}
186 +
187 +func (f *Filecheck) addFileCharts(filePath string, chartsTmpl ...*module.Chart) {
188 + cs := append(module.Charts{}, chartsTmpl...)
189 + charts := cs.Copy()
190 +
191 + for _, chart := range *charts {
192 + chart.ID = fmt.Sprintf(chart.ID, cleanPath(filePath))
193 + chart.Labels = []module.Label{
194 + {Key: "file_path", Value: filePath},
195 + }
196 + for _, dim := range chart.Dims {
197 + dim.ID = fmt.Sprintf(dim.ID, filePath)
198 + }
199 + }
200 +
201 + if err := f.Charts().Add(*charts...); err != nil {
202 + f.Warning(err)
203 + }
204 +}
205 +
206 +func (f *Filecheck) addDirCharts(dirPath string, chartsTmpl ...*module.Chart) {
207 + cs := append(module.Charts{}, chartsTmpl...)
208 + charts := cs.Copy()
209 +
210 + for _, chart := range *charts {
211 + chart.ID = fmt.Sprintf(chart.ID, cleanPath(dirPath))
212 + chart.Labels = []module.Label{
213 + {Key: "dir_path", Value: dirPath},
214 + }
215 + for _, dim := range chart.Dims {
216 + dim.ID = fmt.Sprintf(dim.ID, dirPath)
217 + }
218 + }
219 +
220 + if err := f.Charts().Add(*charts...); err != nil {
221 + f.Warning(err)
222 + }
223 +}
224 +
225 +func (f *Filecheck) removeFileAllCharts(filePath string) {
226 + px := fmt.Sprintf("file_%s_", cleanPath(filePath))
227 + f.removeCharts(func(id string) bool {
228 + return strings.HasPrefix(id, px)
229 + })
230 +}
231 +
232 +func (f *Filecheck) removeFileOtherCharts(filePath string) {
233 + px := fmt.Sprintf("file_%s_", cleanPath(filePath))
234 + f.removeCharts(func(id string) bool {
235 + return strings.HasPrefix(id, px) && !strings.HasSuffix(id, "existence_status")
236 + })
237 +}
238 +
239 +func (f *Filecheck) removeDirAllCharts(dirPath string) {
240 + px := fmt.Sprintf("dir_%s_", cleanPath(dirPath))
241 + f.removeCharts(func(id string) bool {
242 + return strings.HasPrefix(id, px)
243 + })
244 +}
245 +
246 +func (f *Filecheck) removeDirOtherCharts(dirPath string) {
247 + px := fmt.Sprintf("dir_%s_", cleanPath(dirPath))
248 + f.removeCharts(func(id string) bool {
249 + return strings.HasPrefix(id, px) && !strings.HasSuffix(id, "existence_status")
250 + })
251 +}
252 +
253 +func (f *Filecheck) removeCharts(match func(id string) bool) {
254 + for _, chart := range *f.Charts() {
255 + if match(chart.ID) {
256 + chart.MarkRemove()
257 + chart.MarkNotCreated()
258 + }
259 + }
260 +}
261 +
262 +func cleanPath(path string) string {
263 + path = strings.ReplaceAll(path, " ", "_")
264 + path = strings.ReplaceAll(path, ".", "_")
265 + return path
266 +}
src/go/collectors/go.d.plugin/modules/filecheck/collect.go
+24 -24
@@ -3,38 +3,38 @@
3 package filecheck
4
5 import (
6 - "regexp"
7 - "runtime"
8 - "strings"
6 + "errors"
7 + "io/fs"
8 + "os"
9 )
10
11 -func (fc *Filecheck) collect() (map[string]int64, error) {
12 - ms := make(map[string]int64)
11 +func (f *Filecheck) collect() (map[string]int64, error) {
12 + mx := make(map[string]int64)
13
14 - fc.collectFiles(ms)
15 - fc.collectDirs(ms)
14 + f.collectFiles(mx)
15 + f.collectDirs(mx)
16
17 - return ms, nil
17 + return mx, nil
18 }
19
20 -func hasMeta(path string) bool {
21 - magicChars := `*?[`
22 - if runtime.GOOS != "windows" {
23 - magicChars = `*?[\`
24 - }
25 - return strings.ContainsAny(path, magicChars)
20 +type statInfo struct {
21 + path string
22 + exists bool
23 + fi fs.FileInfo
24 }
25
28 -func removeDuplicates(s []string) []string {
29 - set := make(map[string]bool, len(s))
30 - uniq := s[:0]
31 - for _, v := range s {
32 - if !set[v] {
33 - set[v] = true
34 - uniq = append(uniq, v)
26 +func getStatInfo(path string) *statInfo {
27 + fi, err := os.Stat(path)
28 + if err != nil {
29 + return &statInfo{
30 + path: path,
31 + exists: !errors.Is(err, fs.ErrNotExist),
32 }
33 }
37 - return uniq
38 -}
34
40 -var reSpace = regexp.MustCompile(`\s`)
35 + return &statInfo{
36 + path: path,
37 + exists: true,
38 + fi: fi,
39 + }
40 +}
src/go/collectors/go.d.plugin/modules/filecheck/collect_dirs.go
+42 -135
@@ -6,177 +6,84 @@ import (
6 "fmt"
7 "os"
8 "path/filepath"
9 - "strings"
9 "time"
11 -
12 - "github.com/netdata/netdata/go/go.d.plugin/agent/module"
10 )
11
15 -func (fc *Filecheck) collectDirs(ms map[string]int64) {
16 - curTime := time.Now()
17 - if time.Since(fc.lastDiscoveryDirs) >= fc.DiscoveryEvery.Duration() {
18 - fc.lastDiscoveryDirs = curTime
19 - fc.curDirs = fc.discoveryDirs()
20 - fc.updateDirsCharts(fc.curDirs)
21 - }
12 +func (f *Filecheck) collectDirs(mx map[string]int64) {
13 + now := time.Now()
14
23 - for _, path := range fc.curDirs {
24 - fc.collectDir(ms, path, curTime)
15 + if f.isTimeToDiscoverDirs(now) {
16 + f.lastDiscDirsTime = now
17 + f.curDirs = f.discoveryDirs()
18 }
26 - ms["num_of_dirs"] = int64(len(fc.curDirs))
27 -}
19
29 -func (fc *Filecheck) collectDir(ms map[string]int64, path string, curTime time.Time) {
30 - info, err := os.Stat(path)
31 - if err != nil {
32 - if os.IsNotExist(err) {
33 - ms[dirDimID(path, "exists")] = 0
34 - } else {
35 - ms[dirDimID(path, "exists")] = 1
36 - }
37 - fc.Debug(err)
38 - return
39 - }
20 + var infos []*statInfo
21
41 - if !info.IsDir() {
42 - return
43 - }
22 + for _, dir := range f.curDirs {
23 + si := getStatInfo(dir)
24 + infos = append(infos, si)
25
45 - ms[dirDimID(path, "exists")] = 1
46 - ms[dirDimID(path, "mtime_ago")] = int64(curTime.Sub(info.ModTime()).Seconds())
47 - if num, err := calcDirNumOfFiles(path); err == nil {
48 - ms[dirDimID(path, "num_of_files")] = int64(num)
49 - }
50 - if fc.Dirs.CollectDirSize {
51 - if size, err := calcDirSize(path); err == nil {
52 - ms[dirDimID(path, "size_bytes")] = size
53 - }
26 + f.collectDir(mx, si, now)
27 }
28 +
29 + f.updateDirCharts(infos)
30 }
31
57 -func (fc *Filecheck) discoveryDirs() (dirs []string) {
58 - for _, path := range fc.Dirs.Include {
59 - if hasMeta(path) {
60 - continue
61 - }
62 - dirs = append(dirs, path)
63 - }
32 +func (f *Filecheck) collectDir(mx map[string]int64, si *statInfo, now time.Time) {
33 + px := fmt.Sprintf("dir_%s_", si.path)
34
65 - for _, path := range fc.Dirs.Include {
66 - if !hasMeta(path) {
67 - continue
68 - }
69 - matches, _ := filepath.Glob(path)
70 - for _, v := range matches {
71 - fi, err := os.Lstat(v)
72 - if err == nil && fi.IsDir() {
73 - dirs = append(dirs, v)
74 - }
75 - }
35 + mx[px+"existence_status_exist"] = 0
36 + mx[px+"existence_status_not_exist"] = 0
37 + if !si.exists {
38 + mx[px+"existence_status_not_exist"] = 1
39 + } else {
40 + mx[px+"existence_status_exist"] = 1
41 }
77 - return removeDuplicates(dirs)
78 -}
42
80 -func (fc *Filecheck) updateDirsCharts(dirs []string) {
81 - set := make(map[string]bool, len(dirs))
82 - for _, path := range dirs {
83 - set[path] = true
84 - if !fc.collectedDirs[path] {
85 - fc.collectedDirs[path] = true
86 - fc.addDirToCharts(path)
87 - }
88 - }
89 - for path := range fc.collectedDirs {
90 - if !set[path] {
91 - delete(fc.collectedDirs, path)
92 - fc.removeDirFromCharts(path)
93 - }
43 + if si.fi == nil || !si.fi.IsDir() {
44 + return
45 }
95 -}
96 -
97 -func (fc *Filecheck) addDirToCharts(path string) {
98 - for _, chart := range *fc.Charts() {
99 - if !strings.HasPrefix(chart.ID, "dir_") {
100 - continue
101 - }
102 -
103 - var id string
104 - switch chart.ID {
105 - case dirExistenceChart.ID:
106 - id = dirDimID(path, "exists")
107 - case dirModTimeChart.ID:
108 - id = dirDimID(path, "mtime_ago")
109 - case dirNumOfFilesChart.ID:
110 - id = dirDimID(path, "num_of_files")
111 - case dirSizeChart.ID:
112 - id = dirDimID(path, "size_bytes")
113 - default:
114 - fc.Warningf("add dimension: couldn't dim id for '%s' chart (dir '%s')", chart.ID, path)
115 - continue
116 - }
46
118 - dim := &module.Dim{ID: id, Name: reSpace.ReplaceAllString(path, "_")}
47 + mx[px+"mtime_ago"] = int64(now.Sub(si.fi.ModTime()).Seconds())
48
120 - if err := chart.AddDim(dim); err != nil {
121 - fc.Warning(err)
122 - continue
49 + if v, err := calcFilesInDir(si.path); err == nil {
50 + mx[px+"files_count"] = v
51 + }
52 + if f.Dirs.CollectDirSize {
53 + if v, err := calcDirSize(si.path); err == nil {
54 + mx[px+"size_bytes"] = v
55 }
124 - chart.MarkNotCreated()
56 }
57 }
58
128 -func (fc *Filecheck) removeDirFromCharts(path string) {
129 - for _, chart := range *fc.Charts() {
130 - if !strings.HasPrefix(chart.ID, "dir_") {
131 - continue
132 - }
133 -
134 - var id string
135 - switch chart.ID {
136 - case dirExistenceChart.ID:
137 - id = dirDimID(path, "exists")
138 - case dirModTimeChart.ID:
139 - id = dirDimID(path, "mtime_ago")
140 - case dirNumOfFilesChart.ID:
141 - id = dirDimID(path, "num_of_files")
142 - case dirSizeChart.ID:
143 - id = dirDimID(path, "size_bytes")
144 - default:
145 - fc.Warningf("remove dimension: couldn't dim id for '%s' chart (dir '%s')", chart.ID, path)
146 - continue
147 - }
148 -
149 - if err := chart.MarkDimRemove(id, true); err != nil {
150 - fc.Warning(err)
151 - continue
152 - }
153 - chart.MarkNotCreated()
154 - }
59 +func (f *Filecheck) discoveryDirs() (dirs []string) {
60 + return discoverFilesOrDirs(f.Dirs.Include, func(v string, fi os.FileInfo) bool {
61 + return fi.IsDir() && !f.dirsFilter.MatchString(v)
62 + })
63 }
64
157 -func dirDimID(path, metric string) string {
158 - return fmt.Sprintf("dir_%s_%s", reSpace.ReplaceAllString(path, "_"), metric)
65 +func (f *Filecheck) isTimeToDiscoverDirs(now time.Time) bool {
66 + return now.After(f.lastDiscDirsTime.Add(f.DiscoveryEvery.Duration()))
67 }
68
161 -func calcDirNumOfFiles(dirpath string) (int, error) {
162 - f, err := os.Open(dirpath)
69 +func calcFilesInDir(dirPath string) (int64, error) {
70 + f, err := os.Open(dirPath)
71 if err != nil {
72 return 0, err
73 }
74 defer func() { _ = f.Close() }()
167 - // TODO: include dirs?
75 names, err := f.Readdirnames(-1)
169 - return len(names), err
76 + return int64(len(names)), err
77 }
78
172 -func calcDirSize(dirpath string) (int64, error) {
79 +func calcDirSize(dirPath string) (int64, error) {
80 var size int64
174 - err := filepath.Walk(dirpath, func(_ string, info os.FileInfo, err error) error {
81 + err := filepath.Walk(dirPath, func(_ string, fi os.FileInfo, err error) error {
82 if err != nil {
83 return err
84 }
178 - if !info.IsDir() {
179 - size += info.Size()
85 + if !fi.IsDir() {
86 + size += fi.Size()
87 }
88 return nil
89 })
src/go/collectors/go.d.plugin/modules/filecheck/collect_files.go
+31 -120
@@ -5,144 +5,55 @@ package filecheck
5 import (
6 "fmt"
7 "os"
8 - "path/filepath"
9 - "strings"
8 "time"
11 -
12 - "github.com/netdata/netdata/go/go.d.plugin/agent/module"
9 )
10
15 -func (fc *Filecheck) collectFiles(ms map[string]int64) {
16 - curTime := time.Now()
17 - if time.Since(fc.lastDiscoveryFiles) >= fc.DiscoveryEvery.Duration() {
18 - fc.lastDiscoveryFiles = curTime
19 - fc.curFiles = fc.discoveryFiles()
20 - fc.updateFilesCharts(fc.curFiles)
21 - }
22 -
23 - for _, path := range fc.curFiles {
24 - fc.collectFile(ms, path, curTime)
25 - }
26 - ms["num_of_files"] = int64(len(fc.curFiles))
27 -}
11 +func (f *Filecheck) collectFiles(mx map[string]int64) {
12 + now := time.Now()
13
29 -func (fc *Filecheck) collectFile(ms map[string]int64, path string, curTime time.Time) {
30 - info, err := os.Stat(path)
31 - if err != nil {
32 - if os.IsNotExist(err) {
33 - ms[fileDimID(path, "exists")] = 0
34 - } else {
35 - ms[fileDimID(path, "exists")] = 1
36 - }
37 - fc.Debug(err)
38 - return
14 + if f.isTimeToDiscoverFiles(now) {
15 + f.lastDiscFilesTime = now
16 + f.curFiles = f.discoverFiles()
17 }
18
41 - if info.IsDir() {
42 - return
43 - }
19 + var infos []*statInfo
20
45 - ms[fileDimID(path, "exists")] = 1
46 - ms[fileDimID(path, "size_bytes")] = info.Size()
47 - ms[fileDimID(path, "mtime_ago")] = int64(curTime.Sub(info.ModTime()).Seconds())
48 -}
21 + for _, file := range f.curFiles {
22 + si := getStatInfo(file)
23
50 -func (fc *Filecheck) discoveryFiles() (files []string) {
51 - for _, path := range fc.Files.Include {
52 - if hasMeta(path) {
53 - continue
54 - }
55 - files = append(files, path)
56 - }
24 + infos = append(infos, si)
25
58 - for _, path := range fc.Files.Include {
59 - if !hasMeta(path) {
60 - continue
61 - }
62 - matches, _ := filepath.Glob(path)
63 - for _, v := range matches {
64 - fi, err := os.Lstat(v)
65 - if err == nil && fi.Mode().IsRegular() {
66 - files = append(files, v)
67 - }
68 - }
26 + f.collectFile(mx, si, now)
27 }
70 - return removeDuplicates(files)
71 -}
28
73 -func (fc *Filecheck) updateFilesCharts(files []string) {
74 - set := make(map[string]bool, len(files))
75 - for _, path := range files {
76 - set[path] = true
77 - if !fc.collectedFiles[path] {
78 - fc.collectedFiles[path] = true
79 - fc.addFileToCharts(path)
80 - }
81 - }
82 - for path := range fc.collectedFiles {
83 - if !set[path] {
84 - delete(fc.collectedFiles, path)
85 - fc.removeFileFromCharts(path)
86 - }
87 - }
29 + f.updateFileCharts(infos)
30 }
31
90 -func (fc *Filecheck) addFileToCharts(path string) {
91 - for _, chart := range *fc.Charts() {
92 - if !strings.HasPrefix(chart.ID, "file_") {
93 - continue
94 - }
95 -
96 - var id string
97 - switch chart.ID {
98 - case fileExistenceChart.ID:
99 - id = fileDimID(path, "exists")
100 - case fileModTimeAgoChart.ID:
101 - id = fileDimID(path, "mtime_ago")
102 - case fileSizeChart.ID:
103 - id = fileDimID(path, "size_bytes")
104 - default:
105 - fc.Warningf("add dimension: couldn't dim id for '%s' chart (file '%s')", chart.ID, path)
106 - continue
107 - }
108 -
109 - dim := &module.Dim{ID: id, Name: reSpace.ReplaceAllString(path, "_")}
32 +func (f *Filecheck) collectFile(mx map[string]int64, si *statInfo, now time.Time) {
33 + px := fmt.Sprintf("file_%s_", si.path)
34
111 - if err := chart.AddDim(dim); err != nil {
112 - fc.Warning(err)
113 - continue
114 - }
115 - chart.MarkNotCreated()
35 + mx[px+"existence_status_exist"] = 0
36 + mx[px+"existence_status_not_exist"] = 0
37 + if !si.exists {
38 + mx[px+"existence_status_not_exist"] = 1
39 + } else {
40 + mx[px+"existence_status_exist"] = 1
41 }
117 -}
42
119 -func (fc *Filecheck) removeFileFromCharts(path string) {
120 - for _, chart := range *fc.Charts() {
121 - if !strings.HasPrefix(chart.ID, "file_") {
122 - continue
123 - }
43 + if si.fi == nil || !si.fi.Mode().IsRegular() {
44 + return
45 + }
46
125 - var id string
126 - switch chart.ID {
127 - case fileExistenceChart.ID:
128 - id = fileDimID(path, "exists")
129 - case fileModTimeAgoChart.ID:
130 - id = fileDimID(path, "mtime_ago")
131 - case fileSizeChart.ID:
132 - id = fileDimID(path, "size_bytes")
133 - default:
134 - fc.Warningf("remove dimension: couldn't dim id for '%s' chart (file '%s')", chart.ID, path)
135 - continue
136 - }
47 + mx[px+"mtime_ago"] = int64(now.Sub(si.fi.ModTime()).Seconds())
48 + mx[px+"size_bytes"] = si.fi.Size()
49 +}
50
138 - if err := chart.MarkDimRemove(id, true); err != nil {
139 - fc.Warning(err)
140 - continue
141 - }
142 - chart.MarkNotCreated()
143 - }
51 +func (f *Filecheck) discoverFiles() (files []string) {
52 + return discoverFilesOrDirs(f.Files.Include, func(absPath string, fi os.FileInfo) bool {
53 + return fi.Mode().IsRegular() && !f.filesFilter.MatchString(absPath)
54 + })
55 }
56
146 -func fileDimID(path, metric string) string {
147 - return fmt.Sprintf("file_%s_%s", reSpace.ReplaceAllString(path, "_"), metric)
57 +func (f *Filecheck) isTimeToDiscoverFiles(now time.Time) bool {
58 + return now.After(f.lastDiscFilesTime.Add(f.DiscoveryEvery.Duration()))
59 }
src/go/collectors/go.d.plugin/modules/filecheck/config_schema.json
+51 -8
@@ -11,6 +11,13 @@
11 "minimum": 1,
12 "default": 1
13 },
14 + "discovery_every": {
15 + "title": "Scan interval",
16 + "description": "Scan frequency interval (seconds) for files and directories with patterns (globs) in their paths.",
17 + "type": "integer",
18 + "minimum": 1,
19 + "default": 60
20 + },
21 "files": {
22 "title": "File selector",
23 "description": "Configuration for monitoring specific files. If left empy, no files will be monitored.",
@@ -52,12 +59,6 @@
59 "include"
60 ]
61 },
55 - "collect_dir_size": {
56 - "title": "Collect directory size",
57 - "description": "Enable the collection of directory sizes for each monitored directory. Enabling this option may introduce additional overhead on both Netdata and the host system, particularly if directories contain a large number of subdirectories and files.",
58 - "type": "boolean",
59 - "default": false
60 - },
62 "dirs": {
63 "title": "Directory selector",
64 "description": "Configuration for monitoring specific directories. If left empy, no directories will be monitored.",
@@ -66,6 +67,12 @@
67 "null"
68 ],
69 "properties": {
70 + "collect_dir_size": {
71 + "title": "Collect directory size",
72 + "description": "Enable the collection of directory sizes for each monitored directory. **Enabling this option may introduce additional overhead** on both Netdata and the host system, particularly if directories contain a large number of subdirectories and files.",
73 + "type": "boolean",
74 + "default": false
75 + },
76 "include": {
77 "title": "Include",
78 "description": "Include directories that match any of the specified include [patterns](https://golang.org/pkg/path/filepath/#Match).",
@@ -109,13 +116,49 @@
116 "uiOptions": {
117 "fullPage": true
118 },
119 + "ui:flavour": "tabs",
120 + "ui:options": {
121 + "tabs": [
122 + {
123 + "title": "Base",
124 + "fields": [
125 + "update_every",
126 + "discovery_every"
127 + ]
128 + },
129 + {
130 + "title": "Files",
131 + "fields": [
132 + "files"
133 + ]
134 + },
135 + {
136 + "title": "Directories",
137 + "fields": [
138 + "dirs"
139 + ]
140 + }
141 + ]
142 + },
143 "files": {
144 "ui:help": "The logic for inclusion and exclusion is as follows: `(include1 OR include2) AND !(exclude1 OR exclude2)`.",
114 - "ui:collapsible": true
145 + "ui:collapsible": true,
146 + "include": {
147 + "ui:listFlavour": "list"
148 + },
149 + "exclude": {
150 + "ui:listFlavour": "list"
151 + }
152 },
153 "dirs": {
154 "ui:help": "The logic for inclusion and exclusion is as follows: `(include1 OR include2) AND !(exclude1 OR exclude2)`.",
118 - "ui:collapsible": true
155 + "ui:collapsible": true,
156 + "include": {
157 + "ui:listFlavour": "list"
158 + },
159 + "exclude": {
160 + "ui:listFlavour": "list"
161 + }
162 }
163 }
164 }
src/go/collectors/go.d.plugin/modules/filecheck/discover.go new
+43
@@ -0,0 +1,43 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +package filecheck
4 +
5 +import (
6 + "os"
7 + "path/filepath"
8 + "runtime"
9 + "slices"
10 + "strings"
11 +)
12 +
13 +func discoverFilesOrDirs(includePaths []string, fn func(absPath string, fi os.FileInfo) bool) []string {
14 + var paths []string
15 +
16 + for _, path := range includePaths {
17 + if !hasMeta(path) {
18 + paths = append(paths, path)
19 + continue
20 + }
21 +
22 + ps, _ := filepath.Glob(path)
23 + for _, path := range ps {
24 + if fi, err := os.Lstat(path); err == nil && fn(path, fi) {
25 + paths = append(paths, path)
26 + }
27 + }
28 +
29 + }
30 +
31 + slices.Sort(paths)
32 + paths = slices.Compact(paths)
33 +
34 + return paths
35 +}
36 +
37 +func hasMeta(path string) bool {
38 + magicChars := `*?[`
39 + if runtime.GOOS != "windows" {
40 + magicChars = `*?[\`
41 + }
42 + return strings.ContainsAny(path, magicChars)
43 +}
src/go/collectors/go.d.plugin/modules/filecheck/filecheck.go
+41 -32
@@ -7,6 +7,7 @@ import (
7 "time"
8
9 "github.com/netdata/netdata/go/go.d.plugin/agent/module"
10 + "github.com/netdata/netdata/go/go.d.plugin/pkg/matcher"
11 "github.com/netdata/netdata/go/go.d.plugin/pkg/web"
12 )
13
@@ -26,14 +27,13 @@ func init() {
27 func New() *Filecheck {
28 return &Filecheck{
29 Config: Config{
29 - DiscoveryEvery: web.Duration(time.Second * 30),
30 + DiscoveryEvery: web.Duration(time.Minute * 1),
31 Files: filesConfig{},
31 - Dirs: dirsConfig{
32 - CollectDirSize: true,
33 - },
32 + Dirs: dirsConfig{CollectDirSize: false},
33 },
35 - collectedFiles: make(map[string]bool),
36 - collectedDirs: make(map[string]bool),
34 + charts: &module.Charts{},
35 + seenFiles: newSeenItems(),
36 + seenDirs: newSeenItems(),
37 }
38 }
39
@@ -61,58 +61,67 @@ type Filecheck struct {
61
62 charts *module.Charts
63
64 - lastDiscoveryFiles time.Time
65 - curFiles []string
66 - collectedFiles map[string]bool
64 + filesFilter matcher.Matcher
65 + lastDiscFilesTime time.Time
66 + curFiles []string
67 + seenFiles *seenItems
68
68 - lastDiscoveryDirs time.Time
69 - curDirs []string
70 - collectedDirs map[string]bool
69 + dirsFilter matcher.Matcher
70 + lastDiscDirsTime time.Time
71 + curDirs []string
72 + seenDirs *seenItems
73 }
74
73 -func (fc *Filecheck) Configuration() any {
74 - return fc.Config
75 +func (f *Filecheck) Configuration() any {
76 + return f.Config
77 }
78
77 -func (fc *Filecheck) Init() error {
78 - err := fc.validateConfig()
79 +func (f *Filecheck) Init() error {
80 + err := f.validateConfig()
81 if err != nil {
80 - fc.Errorf("error on validating config: %v", err)
82 + f.Errorf("config validation: %v", err)
83 return err
84 }
85
84 - charts, err := fc.initCharts()
86 + ff, err := f.initFilesFilter()
87 if err != nil {
86 - fc.Errorf("error on charts initialization: %v", err)
88 + f.Errorf("files filter initialization: %v", err)
89 return err
90 }
89 - fc.charts = charts
91 + f.filesFilter = ff
92
91 - fc.Debugf("monitored files: %v", fc.Files.Include)
92 - fc.Debugf("monitored dirs: %v", fc.Dirs.Include)
93 + df, err := f.initDirsFilter()
94 + if err != nil {
95 + f.Errorf("dirs filter initialization: %v", err)
96 + return err
97 + }
98 + f.dirsFilter = df
99 +
100 + f.Debugf("monitored files: %v", f.Files.Include)
101 + f.Debugf("monitored dirs: %v", f.Dirs.Include)
102
103 return nil
104 }
105
97 -func (fc *Filecheck) Check() error {
106 +func (f *Filecheck) Check() error {
107 return nil
108 }
109
101 -func (fc *Filecheck) Charts() *module.Charts {
102 - return fc.charts
110 +func (f *Filecheck) Charts() *module.Charts {
111 + return f.charts
112 }
113
105 -func (fc *Filecheck) Collect() map[string]int64 {
106 - ms, err := fc.collect()
114 +func (f *Filecheck) Collect() map[string]int64 {
115 + mx, err := f.collect()
116 if err != nil {
108 - fc.Error(err)
117 + f.Error(err)
118 }
119
111 - if len(ms) == 0 {
120 + if len(mx) == 0 {
121 return nil
122 }
114 - return ms
115 -}
123
117 -func (fc *Filecheck) Cleanup() {
124 + return mx
125 }
126 +
127 +func (f *Filecheck) Cleanup() {}
src/go/collectors/go.d.plugin/modules/filecheck/filecheck_test.go
+73 -83
@@ -37,22 +37,22 @@ func TestFilecheck_Cleanup(t *testing.T) {
37
38 func TestFilecheck_Init(t *testing.T) {
39 tests := map[string]struct {
40 - config Config
41 - wantNumOfCharts int
42 - wantFail bool
40 + config Config
41 + wantFail bool
42 }{
43 "default": {
45 - config: New().Config,
44 wantFail: true,
45 + config: New().Config,
46 },
47 "empty files->include and dirs->include": {
48 + wantFail: true,
49 config: Config{
50 Files: filesConfig{},
51 Dirs: dirsConfig{},
52 },
53 - wantFail: true,
53 },
54 "files->include and dirs->include": {
55 + wantFail: false,
56 config: Config{
57 Files: filesConfig{
58 Include: []string{
@@ -68,9 +68,9 @@ func TestFilecheck_Init(t *testing.T) {
68 CollectDirSize: true,
69 },
70 },
71 - wantNumOfCharts: len(fileCharts) + len(dirCharts),
71 },
72 "only files->include": {
73 + wantFail: false,
74 config: Config{
75 Files: filesConfig{
76 Include: []string{
@@ -79,9 +79,9 @@ func TestFilecheck_Init(t *testing.T) {
79 },
80 },
81 },
82 - wantNumOfCharts: len(fileCharts),
82 },
83 "only dirs->include": {
84 + wantFail: false,
85 config: Config{
86 Dirs: dirsConfig{
87 Include: []string{
@@ -91,7 +91,6 @@ func TestFilecheck_Init(t *testing.T) {
91 CollectDirSize: true,
92 },
93 },
94 - wantNumOfCharts: len(dirCharts),
94 },
95 }
96
@@ -104,7 +103,6 @@ func TestFilecheck_Init(t *testing.T) {
103 assert.Error(t, fc.Init())
104 } else {
105 require.NoError(t, fc.Init())
107 - assert.Equal(t, test.wantNumOfCharts, len(*fc.Charts()))
106 }
107 })
108 }
@@ -142,98 +140,98 @@ func TestFilecheck_Collect(t *testing.T) {
140 "collect files": {
141 prepare: prepareFilecheckFiles,
142 wantCollected: map[string]int64{
145 - "file_testdata/empty_file.log_exists": 1,
146 - "file_testdata/empty_file.log_mtime_ago": 5081,
147 - "file_testdata/empty_file.log_size_bytes": 0,
148 - "file_testdata/file.log_exists": 1,
149 - "file_testdata/file.log_mtime_ago": 4161,
150 - "file_testdata/file.log_size_bytes": 5707,
151 - "file_testdata/non_existent_file.log_exists": 0,
152 - "num_of_files": 3,
153 - "num_of_dirs": 0,
143 + "file_testdata/empty_file.log_existence_status_exist": 1,
144 + "file_testdata/empty_file.log_existence_status_not_exist": 0,
145 + "file_testdata/empty_file.log_mtime_ago": 517996,
146 + "file_testdata/empty_file.log_size_bytes": 0,
147 + "file_testdata/file.log_existence_status_exist": 1,
148 + "file_testdata/file.log_existence_status_not_exist": 0,
149 + "file_testdata/file.log_mtime_ago": 517996,
150 + "file_testdata/file.log_size_bytes": 5707,
151 + "file_testdata/non_existent_file.log_existence_status_exist": 0,
152 + "file_testdata/non_existent_file.log_existence_status_not_exist": 1,
153 },
154 },
155 "collect files filepath pattern": {
156 prepare: prepareFilecheckGlobFiles,
157 wantCollected: map[string]int64{
159 - "file_testdata/empty_file.log_exists": 1,
160 - "file_testdata/empty_file.log_mtime_ago": 5081,
161 - "file_testdata/empty_file.log_size_bytes": 0,
162 - "file_testdata/file.log_exists": 1,
163 - "file_testdata/file.log_mtime_ago": 4161,
164 - "file_testdata/file.log_size_bytes": 5707,
165 - "num_of_files": 2,
166 - "num_of_dirs": 0,
158 + "file_testdata/empty_file.log_existence_status_exist": 1,
159 + "file_testdata/empty_file.log_existence_status_not_exist": 0,
160 + "file_testdata/empty_file.log_mtime_ago": 517985,
161 + "file_testdata/empty_file.log_size_bytes": 0,
162 + "file_testdata/file.log_existence_status_exist": 1,
163 + "file_testdata/file.log_existence_status_not_exist": 0,
164 + "file_testdata/file.log_mtime_ago": 517985,
165 + "file_testdata/file.log_size_bytes": 5707,
166 },
167 },
168 "collect only non existent files": {
169 prepare: prepareFilecheckNonExistentFiles,
170 wantCollected: map[string]int64{
172 - "file_testdata/non_existent_file.log_exists": 0,
173 - "num_of_files": 1,
174 - "num_of_dirs": 0,
171 + "file_testdata/non_existent_file.log_existence_status_exist": 0,
172 + "file_testdata/non_existent_file.log_existence_status_not_exist": 1,
173 },
174 },
175 "collect dirs": {
176 prepare: prepareFilecheckDirs,
177 wantCollected: map[string]int64{
180 - "dir_testdata/dir_exists": 1,
181 - "dir_testdata/dir_mtime_ago": 4087,
182 - "dir_testdata/dir_num_of_files": 3,
183 - "dir_testdata/dir_size_bytes": 8160,
184 - "dir_testdata/non_existent_dir_exists": 0,
185 - "num_of_files": 0,
186 - "num_of_dirs": 2,
178 + "dir_testdata/dir_existence_status_exist": 1,
179 + "dir_testdata/dir_existence_status_not_exist": 0,
180 + "dir_testdata/dir_files_count": 3,
181 + "dir_testdata/dir_mtime_ago": 517914,
182 + "dir_testdata/non_existent_dir_existence_status_exist": 0,
183 + "dir_testdata/non_existent_dir_existence_status_not_exist": 1,
184 },
185 },
186 "collect dirs filepath pattern": {
187 prepare: prepareFilecheckGlobDirs,
188 wantCollected: map[string]int64{
192 - "dir_testdata/dir_exists": 1,
193 - "dir_testdata/dir_mtime_ago": 4087,
194 - "dir_testdata/dir_num_of_files": 3,
195 - "dir_testdata/dir_size_bytes": 8160,
196 - "dir_testdata/non_existent_dir_exists": 0,
197 - "num_of_files": 0,
198 - "num_of_dirs": 2,
189 + "dir_testdata/dir_existence_status_exist": 1,
190 + "dir_testdata/dir_existence_status_not_exist": 0,
191 + "dir_testdata/dir_files_count": 3,
192 + "dir_testdata/dir_mtime_ago": 517902,
193 + "dir_testdata/non_existent_dir_existence_status_exist": 0,
194 + "dir_testdata/non_existent_dir_existence_status_not_exist": 1,
195 },
196 },
197 "collect dirs w/o size": {
198 prepare: prepareFilecheckDirsWithoutSize,
199 wantCollected: map[string]int64{
204 - "dir_testdata/dir_exists": 1,
205 - "dir_testdata/dir_mtime_ago": 4087,
206 - "dir_testdata/dir_num_of_files": 3,
207 - "dir_testdata/non_existent_dir_exists": 0,
208 - "num_of_files": 0,
209 - "num_of_dirs": 2,
200 + "dir_testdata/dir_existence_status_exist": 1,
201 + "dir_testdata/dir_existence_status_not_exist": 0,
202 + "dir_testdata/dir_files_count": 3,
203 + "dir_testdata/dir_mtime_ago": 517892,
204 + "dir_testdata/non_existent_dir_existence_status_exist": 0,
205 + "dir_testdata/non_existent_dir_existence_status_not_exist": 1,
206 },
207 },
208 "collect only non existent dirs": {
209 prepare: prepareFilecheckNonExistentDirs,
210 wantCollected: map[string]int64{
215 - "dir_testdata/non_existent_dir_exists": 0,
216 - "num_of_files": 0,
217 - "num_of_dirs": 1,
211 + "dir_testdata/non_existent_dir_existence_status_exist": 0,
212 + "dir_testdata/non_existent_dir_existence_status_not_exist": 1,
213 },
214 },
215 "collect files and dirs": {
216 prepare: prepareFilecheckFilesDirs,
217 wantCollected: map[string]int64{
223 - "dir_testdata/dir_exists": 1,
224 - "dir_testdata/dir_mtime_ago": 4120,
225 - "dir_testdata/dir_num_of_files": 3,
226 - "dir_testdata/dir_size_bytes": 8160,
227 - "dir_testdata/non_existent_dir_exists": 0,
228 - "file_testdata/empty_file.log_exists": 1,
229 - "file_testdata/empty_file.log_mtime_ago": 5176,
230 - "file_testdata/empty_file.log_size_bytes": 0,
231 - "file_testdata/file.log_exists": 1,
232 - "file_testdata/file.log_mtime_ago": 4256,
233 - "file_testdata/file.log_size_bytes": 5707,
234 - "file_testdata/non_existent_file.log_exists": 0,
235 - "num_of_files": 3,
236 - "num_of_dirs": 2,
218 + "dir_testdata/dir_existence_status_exist": 1,
219 + "dir_testdata/dir_existence_status_not_exist": 0,
220 + "dir_testdata/dir_files_count": 3,
221 + "dir_testdata/dir_mtime_ago": 517858,
222 + "dir_testdata/dir_size_bytes": 8160,
223 + "dir_testdata/non_existent_dir_existence_status_exist": 0,
224 + "dir_testdata/non_existent_dir_existence_status_not_exist": 1,
225 + "file_testdata/empty_file.log_existence_status_exist": 1,
226 + "file_testdata/empty_file.log_existence_status_not_exist": 0,
227 + "file_testdata/empty_file.log_mtime_ago": 517858,
228 + "file_testdata/empty_file.log_size_bytes": 0,
229 + "file_testdata/file.log_existence_status_exist": 1,
230 + "file_testdata/file.log_existence_status_not_exist": 0,
231 + "file_testdata/file.log_mtime_ago": 517858,
232 + "file_testdata/file.log_size_bytes": 5707,
233 + "file_testdata/non_existent_file.log_existence_status_exist": 0,
234 + "file_testdata/non_existent_file.log_existence_status_not_exist": 1,
235 },
236 },
237 }
@@ -243,31 +241,23 @@ func TestFilecheck_Collect(t *testing.T) {
241 fc := test.prepare()
242 require.NoError(t, fc.Init())
243
246 - collected := fc.Collect()
244 + mx := fc.Collect()
245
248 - copyModTime(test.wantCollected, collected)
249 - assert.Equal(t, test.wantCollected, collected)
250 - ensureCollectedHasAllChartsDimsVarsIDs(t, fc, collected)
246 + copyModTime(test.wantCollected, mx)
247 + assert.Equal(t, test.wantCollected, mx)
248 + testMetricsHasAllChartsDims(t, fc, mx)
249 })
250 }
251 }
252
255 -func ensureCollectedHasAllChartsDimsVarsIDs(t *testing.T, fc *Filecheck, collected map[string]int64) {
256 - // TODO: check other charts
253 +func testMetricsHasAllChartsDims(t *testing.T, fc *Filecheck, mx map[string]int64) {
254 for _, chart := range *fc.Charts() {
255 if chart.Obsolete {
256 continue
257 }
261 - switch chart.ID {
262 - case fileExistenceChart.ID, dirExistenceChart.ID:
263 - for _, dim := range chart.Dims {
264 - _, ok := collected[dim.ID]
265 - assert.Truef(t, ok, "collected metrics has no data for dim '%s' chart '%s'", dim.ID, chart.ID)
266 - }
267 - for _, v := range chart.Vars {
268 - _, ok := collected[v.ID]
269 - assert.Truef(t, ok, "collected metrics has no data for var '%s' chart '%s'", v.ID, chart.ID)
270 - }
258 + for _, dim := range chart.Dims {
259 + _, ok := mx[dim.ID]
260 + assert.Truef(t, ok, "mx metrics has no data for dim '%s' chart '%s'", dim.ID, chart.ID)
261 }
262 }
263 }
@@ -322,7 +312,6 @@ func prepareFilecheckDirsWithoutSize() *Filecheck {
312 "testdata/dir",
313 "testdata/non_existent_dir",
314 }
325 - fc.Config.Dirs.CollectDirSize = false
315 return fc
316 }
317
@@ -336,6 +325,7 @@ func prepareFilecheckNonExistentDirs() *Filecheck {
325
326 func prepareFilecheckFilesDirs() *Filecheck {
327 fc := New()
328 + fc.Config.Dirs.CollectDirSize = true
329 fc.Config.Files.Include = []string{
330 "testdata/empty_file.log",
331 "testdata/file.log",
src/go/collectors/go.d.plugin/modules/filecheck/init.go
+17 -21
@@ -5,38 +5,34 @@ package filecheck
5 import (
6 "errors"
7
8 - "github.com/netdata/netdata/go/go.d.plugin/agent/module"
8 + "github.com/netdata/netdata/go/go.d.plugin/pkg/matcher"
9 )
10
11 -func (fc *Filecheck) validateConfig() error {
12 - if len(fc.Files.Include) == 0 && len(fc.Dirs.Include) == 0 {
11 +func (f *Filecheck) validateConfig() error {
12 + if len(f.Files.Include) == 0 && len(f.Dirs.Include) == 0 {
13 return errors.New("both 'files->include' and 'dirs->include' are empty")
14 }
15 return nil
16 }
17
18 -func (fc *Filecheck) initCharts() (*module.Charts, error) {
19 - charts := &module.Charts{}
18 +func (f *Filecheck) initFilesFilter() (matcher.Matcher, error) {
19 + return newFilter(f.Files.Exclude)
20 +}
21
21 - if len(fc.Files.Include) > 0 {
22 - if err := charts.Add(*fileCharts.Copy()...); err != nil {
23 - return nil, err
24 - }
25 - }
22 +func (f *Filecheck) initDirsFilter() (matcher.Matcher, error) {
23 + return newFilter(f.Dirs.Exclude)
24 +}
25 +
26 +func newFilter(patterns []string) (matcher.Matcher, error) {
27 + filter := matcher.FALSE()
28
27 - if len(fc.Dirs.Include) > 0 {
28 - if err := charts.Add(*dirCharts.Copy()...); err != nil {
29 + for _, s := range patterns {
30 + m, err := matcher.NewGlobMatcher(s)
31 + if err != nil {
32 return nil, err
33 }
31 - if !fc.Dirs.CollectDirSize {
32 - if err := charts.Remove(dirSizeChart.ID); err != nil {
33 - return nil, err
34 - }
35 - }
34 + filter = matcher.Or(filter, m)
35 }
36
38 - if len(*charts) == 0 {
39 - return nil, errors.New("empty charts")
40 - }
41 - return charts, nil
37 + return filter, nil
38 }
src/go/collectors/go.d.plugin/modules/filecheck/metadata.yaml
+39 -29
@@ -60,7 +60,7 @@ modules:
60 default_value: 0
61 required: false
62 - name: files
63 - description: Files matching the selector will be monitored.
63 + description: List of files to monitor.
64 default_value: ""
65 required: true
66 detailed_description: |
@@ -140,49 +140,59 @@ modules:
140 description: ""
141 availability: []
142 scopes:
143 - - name: global
144 - description: TBD
145 - labels: []
143 + - name: file
144 + description: These metrics refer to the File.
145 + labels:
146 + - name: file_path
147 + description: File absolute path
148 metrics:
147 - - name: filecheck.file_existence
148 - description: 'File Existence (0: not exists, 1: exists)'
149 - unit: boolean
149 + - name: filecheck.file_existence_status
150 + description: File existence
151 + unit: status
152 chart_type: line
153 dimensions:
152 - - name: a dimension per file
153 - - name: filecheck.file_mtime_ago
154 - description: File Time Since the Last Modification
154 + - name: exist
155 + - name: not_exist
156 + - name: filecheck.file_modification_time_ago
157 + description: File time since the last modification
158 unit: seconds
159 chart_type: line
160 dimensions:
158 - - name: a dimension per file
159 - - name: filecheck.file_size
160 - description: File Size
161 + - name: mtime_ago
162 + - name: filecheck.file_size_bytes
163 + description: File size
164 unit: bytes
165 chart_type: line
166 dimensions:
164 - - name: a dimension per file
165 - - name: filecheck.dir_existence
166 - description: 'Dir Existence (0: not exists, 1: exists)'
167 - unit: boolean
167 + - name: size
168 + - name: directory
169 + description: These metrics refer to the Directory.
170 + labels:
171 + - name: dir_path
172 + description: Directory absolute path
173 + metrics:
174 + - name: filecheck.dir_existence_status
175 + description: Directory existence
176 + unit: status
177 chart_type: line
178 dimensions:
170 - - name: a dimension per directory
171 - - name: filecheck.dir_mtime_ago
172 - description: Dir Time Since the Last Modification
179 + - name: exist
180 + - name: not_exist
181 + - name: filecheck.dir_modification_time_ago
182 + description: Directory time since the last modification
183 unit: seconds
184 chart_type: line
185 dimensions:
176 - - name: a dimension per directory
177 - - name: filecheck.dir_num_of_files
178 - description: Dir Number of Files
179 - unit: files
186 + - name: mtime_ago
187 + - name: filecheck.dir_size_bytes
188 + description: Directory size
189 + unit: bytes
190 chart_type: line
191 dimensions:
182 - - name: a dimension per directory
183 - - name: filecheck.dir_size
184 - description: Dir Size
185 - unit: bytes
192 + - name: size
193 + - name: filecheck.dir_files count
194 + description: Directory files count
195 + unit: files
196 chart_type: line
197 dimensions:
188 - - name: a dimension per directory
198 + - name: files