@cryptotaxi247 / netdata-1 / commits / e8ec31c39

feat(go.d): add generic sql collector (#21281)

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Ilya Mashchenko committed Nov 17, 2025 at 13:54 UTC e8ec31c390e4751905f16bd4f4e063c6f995b499

14 files changed +2590 -10

src/go/go.mod

@@ -69,6 +69,7 @@ require (

69 require (

70 github.com/alexbrainman/odbc v0.0.0-20250601004241-49e6b2bc0cf0

71 github.com/ibm-messaging/mq-golang/v5 v5.6.6

72 + github.com/microsoft/go-mssqldb v1.9.4

73 gopkg.in/yaml.v3 v3.0.1

74 )

@@ -100,6 +101,8 @@ require (

101 github.com/go-openapi/jsonreference v0.21.0 // indirect

102 github.com/go-openapi/swag v0.23.0 // indirect

103 github.com/gogo/protobuf v1.3.2 // indirect

104 + github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect

105 + github.com/golang-sql/sqlexp v0.1.0 // indirect

106 github.com/golang/protobuf v1.5.4 // indirect

107 github.com/golang/snappy v0.0.4 // indirect

108 github.com/google/certificate-transparency-go v1.1.7 // indirect

src/go/go.sum

+20 -10

@@ -9,18 +9,22 @@ dario.cat/mergo v1.0.1 h1:Ra4+bf83h2ztPIQYNP99R6m+Y7KfnARDfID+a+vLl4s=

9 dario.cat/mergo v1.0.1/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=

10 filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=

11 filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=

12 -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0 h1:g0EZJwz7xkXQiZAI5xi9f3WWFYBlX1CPTrR+NDToRkQ=

13 -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.17.0/go.mod h1:XCW7KnZet0Opnr7HccfUw1PLc4CjHqpcaxW8DHklNkQ=

14 -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1 h1:1mvYtZfWQAnwNah/C+Z+Jb9rQH95LPE2vlmMuWAHJk8=

15 -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.1/go.mod h1:75I/mXtme1JyWFtz8GocPHVFyH421IBoZErnO16dd0k=

16 -github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=

17 -github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=

12 +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U=

13 +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM=

14 +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1 h1:B+blDbyVIG3WaikNxPnhPiJ1MThR03b3vKGtER95TP4=

15 +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.10.1/go.mod h1:JdM5psgjfBf5fo2uWOZhflPWyDBZ/O/CNAH9CtsuZE4=

16 +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4=

17 +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA=

18 +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1 h1:Wgf5rZba3YZqeTNJPtvqZoBu1sBN/L4sry+u2U3Y75w=

19 +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/azkeys v1.3.1/go.mod h1:xxCBG/f/4Vbmh2XQJBsOmNdxWUY5j/s27jujKPbQf14=

20 +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1 h1:bFWuoEKg+gImo7pvkiQEFAc8ocibADgXeiLAxWhWmkI=

21 +github.com/Azure/azure-sdk-for-go/sdk/security/keyvault/internal v1.1.1/go.mod h1:Vih/3yc6yac2JzU4hzpaDupBJP0Flaia9rXXrU8xyww=

22 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=

23 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=

24 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8=

25 github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=

22 -github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2 h1:kYRSnvJju5gYVyhkij+RTJ/VR6QIUaCfWeaFm2ycsjQ=

23 -github.com/AzureAD/microsoft-authentication-library-for-go v1.3.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=

26 +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs=

27 +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI=

28 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=

29 github.com/DATA-DOG/go-sqlmock v1.5.2 h1:OcvFkGmslmlZibjAjaHm3L//6LiuBgolP7OputlJIzU=

30 github.com/DATA-DOG/go-sqlmock v1.5.2/go.mod h1:88MAG/4G7SMwSE3CeA0ZKzrT5CiOU3OJ+JlNzwDqpNU=

@@ -147,8 +151,12 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=

151 github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=

152 github.com/gohugoio/hashstructure v0.6.0 h1:7wMB/2CfXoThFYhdWRGv3u3rUM761Cq29CxUW+NltUg=

153 github.com/gohugoio/hashstructure v0.6.0/go.mod h1:lapVLk9XidheHG1IQ4ZSbyYrXcaILU1ZEP/+vno5rBQ=

150 -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk=

151 -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=

154 +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8=

155 +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=

156 +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA=

157 +github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=

158 +github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=

159 +github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=

160 github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=

161 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=

162 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

@@ -325,6 +333,8 @@ github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/

333 github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=

334 github.com/mdlayher/socket v0.4.1 h1:eM9y2/jlbs1M615oshPQOHZzj6R6wMT7bX5NPiQvn2U=

335 github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=

336 +github.com/microsoft/go-mssqldb v1.9.4 h1:sHrj3GcdgkxytZ09aZ3+ys72pMeyEXJowT44j74pNgs=

337 +github.com/microsoft/go-mssqldb v1.9.4/go.mod h1:GBbW9ASTiDC+mpgWDGKdm3FnFLTUsLYN3iFL90lQ+PA=

338 github.com/miekg/dns v1.1.68 h1:jsSRkNozw7G/mnmXULynzMNIsgY2dHC8LO6U6Ij2JEA=

339 github.com/miekg/dns v1.1.68/go.mod h1:fujopn7TB3Pu3JM69XaawiU0wqjpL9/8xGop5UrTPps=

340 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE924+mUcZuXKLBHA35U7LN621Bws=

src/go/plugin/go.d/collector/init.go

@@ -102,6 +102,7 @@ import (

102 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/smartctl"

103 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/snmp"

104 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/spigotmc"

105 + _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/sql"

106 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/squid"

107 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/squidlog"

108 _ "github.com/netdata/netdata/go/plugins/plugin/go.d/collector/storcli"

src/go/plugin/go.d/collector/sql/charts.go new

+53

@@ -0,0 +1,53 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +package sql

4 +

5 +import (

6 + "fmt"

7 +

8 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"

9 +)

10 +

11 +const (

12 + prioChart = module.Priority + iota

13 +)

14 +

15 +func (c *Collector) createChart(chartID string, m ConfigMetricBlock, ch ConfigChartConfig, row map[string]string) {

16 + if c.seenCharts[chartID] {

17 + return

18 + }

19 + c.seenCharts[chartID] = true

20 +

21 + chart := &module.Chart{

22 + ID: chartID,

23 + Title: ch.Title,

24 + Units: ch.Units,

25 + Type: module.ChartType(ch.Type),

26 + Ctx: fmt.Sprintf("sql.%s_%s", c.Driver, ch.Context),

27 + Fam: ch.Family,

28 + Priority: prioChart,

29 + }

30 +

31 + for k, v := range c.StaticLabels {

32 + chart.Labels = append(chart.Labels, module.Label{Key: k, Value: v})

33 + }

34 + chart.Labels = append(chart.Labels, module.Label{Key: "driver", Value: c.Driver})

35 +

36 + for _, lf := range m.LabelsFromRow {

37 + if v, ok := row[lf.Source]; ok {

38 + chart.Labels = append(chart.Labels, module.Label{Key: lf.Name, Value: v})

39 + }

40 + }

41 +

42 + for _, d := range ch.Dims {

43 + chart.Dims = append(chart.Dims, &module.Dim{

44 + ID: buildDimID(chartID, d.Name),

45 + Name: d.Name,

46 + Algo: module.DimAlgo(ch.Algorithm),

47 + })

48 + }

49 +

50 + if err := c.Charts().Add(chart); err != nil {

51 + c.Warningf("failed to add chart %q: %v", chartID, err)

52 + }

53 +}

src/go/plugin/go.d/collector/sql/collect.go new

+266

@@ -0,0 +1,266 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +package sql

4 +

5 +import (

6 + "context"

7 + "database/sql"

8 + "fmt"

9 + "slices"

10 + "strconv"

11 + "strings"

12 + "time"

13 +

14 + "github.com/netdata/netdata/go/plugins/plugin/go.d/pkg/metrix"

15 +)

16 +

17 +func (c *Collector) collect(ctx context.Context) (map[string]int64, error) {

18 + if c.db == nil {

19 + if err := c.openConnection(); err != nil {

20 + return nil, err

21 + }

22 + }

23 +

24 + qcache, _, err := c.execReusableQueries(ctx)

25 + if err != nil {

26 + return nil, err

27 + }

28 +

29 + mcache, _, err := c.execMetricQueries(ctx, qcache)

30 + if err != nil {

31 + return nil, err

32 + }

33 +

34 + mx := make(map[string]int64)

35 + if err := c.collectMetrics(mx, mcache); err != nil {

36 + return nil, err

37 + }

38 +

39 + return mx, nil

40 +}

41 +

42 +func (c *Collector) collectMetrics(mx map[string]int64, mcache QueryRowsCache) error {

43 + for i, m := range c.Metrics {

44 + rows, ok := mcache[m.ID]

45 + if !ok {

46 + continue

47 + }

48 +

49 + switch strings.ToLower(m.Mode) {

50 + case "columns", "":

51 + if err := c.collectMetricsModeColumns(mx, m, rows); err != nil {

52 + return fmt.Errorf("metric %q (index %d) columns: %w", m.ID, i, err)

53 + }

54 + case "kv":

55 + if err := c.collectMetricsModeKV(mx, m, rows); err != nil {

56 + return fmt.Errorf("metric %q (index %d) kv: %w", m.ID, i, err)

57 + }

58 + default:

59 + return fmt.Errorf("metric %q (index %d) unknown mode %q", m.ID, i, m.Mode)

60 + }

61 + }

62 + return nil

63 +}

64 +

65 +func (c *Collector) collectMetricsModeColumns(mx map[string]int64, m ConfigMetricBlock, rows []map[string]string) error {

66 + for _, ch := range m.Charts {

67 + for _, row := range rows {

68 + chartID := c.buildChartID(m, ch, row)

69 + if chartID == "" {

70 + continue

71 + }

72 + c.createChart(chartID, m, ch, row)

73 +

74 + for _, d := range ch.Dims {

75 + raw, ok := row[d.Source]

76 + if !ok {

77 + continue

78 + }

79 + id := buildDimID(chartID, d.Name)

80 +

81 + if d.StatusWhen == nil {

82 + mx[id] += toInt64(raw)

83 + } else if v, ok := mx[id]; !ok || v == 0 {

84 + mx[id] = metrix.Bool(c.evalStatusWhen(d.StatusWhen, raw))

85 + }

86 + }

87 + }

88 + }

89 +

90 + return nil

91 +}

92 +

93 +func (c *Collector) collectMetricsModeKV(mx map[string]int64, m ConfigMetricBlock, rows []map[string]string) error {

94 + if m.KVMode == nil {

95 + return nil

96 + }

97 +

98 + nameCol := m.KVMode.NameCol

99 + valCol := m.KVMode.ValueCol

100 +

101 + for _, ch := range m.Charts {

102 + for _, row := range rows {

103 + chartID := c.buildChartID(m, ch, row)

104 + if chartID == "" {

105 + continue

106 + }

107 + c.createChart(chartID, m, ch, row)

108 +

109 + k, ok1 := row[nameCol]

110 + vraw, ok2 := row[valCol]

111 + if !ok1 || !ok2 {

112 + continue

113 + }

114 +

115 + for _, d := range ch.Dims {

116 + if d.Source != k {

117 + continue

118 + }

119 + id := buildDimID(chartID, d.Name)

120 +

121 + if d.StatusWhen == nil {

122 + mx[id] += toInt64(vraw)

123 + } else if v, ok := mx[id]; !ok || v == 0 {

124 + mx[id] = metrix.Bool(c.evalStatusWhen(d.StatusWhen, vraw))

125 + }

126 + }

127 + }

128 + }

129 +

130 + return nil

131 +}

132 +

133 +func (c *Collector) evalStatusWhen(sw *ConfigStatusWhen, value string) bool {

134 + switch {

135 + case sw.Equals != "":

136 + return value == sw.Equals

137 + case len(sw.In) > 0:

138 + return slices.Contains(sw.In, value)

139 + case sw.re != nil:

140 + return sw.re.MatchString(value)

141 + default:

142 + return false

143 + }

144 +}

145 +

146 +func (c *Collector) openConnection() error {

147 + db, err := sql.Open(c.Driver, c.DSN)

148 + if err != nil {

149 + return fmt.Errorf("open %s: %w (dsn=%s)", c.Driver, err, redactDSN(c.DSN))

150 + }

151 +

152 + db.SetConnMaxLifetime(time.Minute * 10)

153 +

154 + ctx, cancel := context.WithTimeout(context.Background(), c.Timeout.Duration())

155 + defer cancel()

156 +

157 + if err := db.PingContext(ctx); err != nil {

158 + _ = db.Close()

159 + return fmt.Errorf("ping %s: %w (dsn=%s)", c.Driver, err, redactDSN(c.DSN))

160 + }

161 +

162 + c.db = db

163 +

164 + return nil

165 +}

166 +

167 +func (c *Collector) buildChartID(m ConfigMetricBlock, ch ConfigChartConfig, row map[string]string) string {

168 + var b strings.Builder

169 + b.Grow(128)

170 +

171 + b.WriteString(c.Driver + "_" + m.ID + "_" + ch.Context)

172 +

173 + for _, lf := range m.LabelsFromRow {

174 + v, ok := row[lf.Source]

175 + if !ok {

176 + return ""

177 + }

178 + b.WriteString("_" + v)

179 + }

180 +

181 + return normalizeID(b.String())

182 +}

183 +

184 +func buildDimID(chartID, dimName string) string {

185 + return normalizeID(chartID + "." + dimName)

186 +}

187 +

188 +var idReplacer = strings.NewReplacer(" ", "_", ".", "_")

189 +

190 +func normalizeID(id string) string {

191 + return strings.ToLower(idReplacer.Replace(id))

192 +}

193 +

194 +func redactDSN(dsn string) string {

195 + if dsn == "" {

196 + return dsn

197 + }

198 +

199 + // Find where authority starts (right after "://", if present)

200 + authStart := 0

201 + if i := strings.Index(dsn, "://"); i != -1 {

202 + authStart = i + 3

203 + }

204 +

205 + // Find the *last* '@' after authority start

206 + rel := strings.LastIndex(dsn[authStart:], "@")

207 + if rel == -1 {

208 + // no userinfo

209 + return dsn

210 + }

211 + at := authStart + rel

212 +

213 + // userinfo is between authority start and '@'

214 + userinfo := dsn[authStart:at]

215 + if userinfo == "" {

216 + // malformed/empty userinfo; leave unchanged

217 + return dsn

218 + }

219 +

220 + // If there's a colon, treat text before first ':' as user and the rest as password.

221 + if colon := strings.IndexByte(userinfo, ':'); colon >= 0 {

222 + user := userinfo[:colon]

223 + // Keep user, redact password

224 + redacted := user + ":****"

225 + return dsn[:authStart] + redacted + dsn[at:]

226 + }

227 +

228 + // No password present -> redact entire username

229 + return dsn[:authStart] + "****" + dsn[at:]

230 +}

231 +

232 +func makeRawBytesSlice(size int) []any {

233 + values := make([]any, size)

234 + for i := range values {

235 + var b sql.RawBytes

236 + values[i] = &b

237 + }

238 + return values

239 +}

240 +

241 +func rawBytesToString(value any) string {

242 + if rb, ok := value.(*sql.RawBytes); ok && rb != nil {

243 + return string(*rb)

244 + }

245 + return ""

246 +}

247 +

248 +// local parser: int -> float -> bool -> 0

249 +func toInt64(s string) int64 {

250 + if s == "" {

251 + return 0

252 + }

253 + if i, err := strconv.ParseInt(s, 10, 64); err == nil {

254 + return i

255 + }

256 + if f, err := strconv.ParseFloat(s, 64); err == nil {

257 + return int64(f)

258 + }

259 + switch strings.ToLower(strings.TrimSpace(s)) {

260 + case "true", "t", "yes", "y", "on", "up":

261 + return 1

262 + case "false", "f", "no", "n", "off", "down":

263 + return 0

264 + }

265 + return 0

266 +}

src/go/plugin/go.d/collector/sql/collector.go new

+91

@@ -0,0 +1,91 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +package sql

4 +

5 +import (

6 + "context"

7 + "database/sql"

8 + _ "embed"

9 + "errors"

10 + "time"

11 +

12 + "github.com/netdata/netdata/go/plugins/pkg/confopt"

13 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"

14 +)

15 +

16 +//go:embed "config_schema.json"

17 +var configSchema string

18 +

19 +func init() {

20 + module.Register("sql", module.Creator{

21 + Create: func() module.Module { return New() },

22 + JobConfigSchema: configSchema,

23 + Config: func() any { return &Config{} },

24 + })

25 +}

26 +

27 +func New() *Collector {

28 + return &Collector{

29 + Config: Config{

30 + Driver: "mysql",

31 + Timeout: confopt.Duration(time.Second * 5),

32 + },

33 + charts: &module.Charts{},

34 + seenCharts: make(map[string]bool),

35 + skipValues: make(map[string]bool),

36 + }

37 +}

38 +

39 +type Collector struct {

40 + module.Base

41 + Config `yaml:",inline" json:""`

42 +

43 + charts *module.Charts

44 +

45 + db *sql.DB

46 +

47 + seenCharts map[string]bool

48 + skipValues map[string]bool

49 +}

50 +

51 +func (c *Collector) Configuration() any {

52 + return c.Config

53 +}

54 +

55 +func (c *Collector) Charts() *module.Charts {

56 + return c.charts

57 +}

58 +

59 +func (c *Collector) Init(context.Context) error {

60 + return c.validateConfig()

61 +}

62 +

63 +func (c *Collector) Check(ctx context.Context) error {

64 + mx, err := c.collect(ctx)

65 + if err != nil {

66 + return err

67 + }

68 + if len(mx) == 0 {

69 + return errors.New("no metrics collected")

70 + }

71 + return nil

72 +}

73 +

74 +func (c *Collector) Collect(ctx context.Context) map[string]int64 {

75 + mx, err := c.collect(ctx)

76 + if err != nil {

77 + c.Error(err)

78 + }

79 +

80 + if len(mx) == 0 {

81 + return nil

82 + }

83 + return mx

84 +}

85 +

86 +func (c *Collector) Cleanup(context.Context) {

87 + if c.db != nil {

88 + _ = c.db.Close()

89 + c.db = nil

90 + }

91 +}

src/go/plugin/go.d/collector/sql/collector_test.go new

+681

@@ -0,0 +1,681 @@

1 +package sql

2 +

3 +import (

4 + "context"

5 + "os"

6 + "testing"

7 +

8 + "github.com/DATA-DOG/go-sqlmock"

9 + "github.com/stretchr/testify/assert"

10 + "github.com/stretchr/testify/require"

11 +

12 + "github.com/netdata/netdata/go/plugins/plugin/go.d/agent/module"

13 +)

14 +

15 +var (

16 + dataConfigJSON, _ = os.ReadFile("testdata/config.json")

17 + dataConfigYAML, _ = os.ReadFile("testdata/config.yaml")

18 +)

19 +

20 +func TestCollector_ConfigurationSerialize(t *testing.T) {

21 + module.TestConfigurationSerialize(t, &Collector{}, dataConfigJSON, dataConfigYAML)

22 +}

23 +

24 +func TestCollector_Charts(t *testing.T) {

25 + assert.NotNil(t, New().Charts())

26 +}

27 +

28 +func TestCollector_Cleanup(t *testing.T) {

29 + tests := map[string]func(t *testing.T) (collr *Collector, cleanup func()){

30 + "db connection not initialized": func(t *testing.T) (collr *Collector, cleanup func()) {

31 + return New(), func() {}

32 + },

33 + "db connection initialized": func(t *testing.T) (collr *Collector, cleanup func()) {

34 + db, mock, err := sqlmock.New()

35 + require.NoError(t, err)

36 +

37 + mock.ExpectClose()

38 + collr = New()

39 + collr.db = db

40 + cleanup = func() { _ = db.Close() }

41 +

42 + return collr, cleanup

43 + },

44 + }

45 +

46 + for name, prepare := range tests {

47 + t.Run(name, func(t *testing.T) {

48 + collr, cleanup := prepare(t)

49 + defer cleanup()

50 +

51 + assert.NotPanics(t, func() { collr.Cleanup(context.Background()) })

52 + assert.Nil(t, collr.db)

53 + })

54 + }

55 +}

56 +

57 +func TestCollector_Check(t *testing.T) {

58 + const query = "SELECT 1 AS value"

59 +

60 + tests := map[string]struct {

61 + prepareMock func(t *testing.T, m sqlmock.Sqlmock)

62 + wantFail bool

63 + }{

64 + "success when metrics collected": {

65 + wantFail: false,

66 + prepareMock: func(t *testing.T, m sqlmock.Sqlmock) {

67 + rows := sqlmock.NewRows([]string{"value"}).AddRow("1")

68 + m.ExpectQuery(query).WillReturnRows(rows).RowsWillBeClosed()

69 + },

70 + },

71 + "error when query fails": {

72 + wantFail: true,

73 + prepareMock: func(t *testing.T, m sqlmock.Sqlmock) {

74 + m.ExpectQuery(query).WillReturnError(assert.AnError)

75 + },

76 + },

77 + }

78 +

79 + for name, test := range tests {

80 + t.Run(name, func(t *testing.T) {

81 + db, mock, err := sqlmock.New(

82 + sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual),

83 + )

84 + require.NoError(t, err)

85 + defer func() { _ = db.Close() }()

86 +

87 + collr := New()

88 + collr.db = db

89 + collr.Driver = "pgx"

90 + collr.DSN = "postgres://user:pass@localhost/db"

91 + collr.Metrics = []ConfigMetricBlock{

92 + {

93 + ID: "m1",

94 + Mode: "columns",

95 + Query: query,

96 + Charts: []ConfigChartConfig{

97 + {

98 + Title: "test",

99 + Context: "pg.test",

100 + Family: "test",

101 + Units: "events",

102 + Dims: []ConfigDimConfig{

103 + {Name: "value", Source: "value"},

104 + },

105 + },

106 + },

107 + },

108 + }

109 +

110 + require.NoError(t, collr.Init(context.Background()))

111 +

112 + test.prepareMock(t, mock)

113 +

114 + if test.wantFail {

115 + assert.Error(t, collr.Check(context.Background()))

116 + } else {

117 + assert.NoError(t, collr.Check(context.Background()))

118 + }

119 + assert.NoError(t, mock.ExpectationsWereMet())

120 + })

121 + }

122 +}

123 +

124 +func TestCollector_Collect(t *testing.T) {

125 +

126 + type testCase struct {

127 + prepare func(t *testing.T, mock sqlmock.Sqlmock, coll *Collector)

128 + check func(t *testing.T, mx map[string]int64)

129 + }

130 +

131 + tests := map[string]testCase{

132 + "columns: metric columns without labels (bgwriter-like)": {

133 + prepare: func(t *testing.T, mock sqlmock.Sqlmock, coll *Collector) {

134 + query := `

135 +SELECT

136 + checkpoints_timed,

137 + checkpoints_req,

138 + checkpoint_write_time,

139 + checkpoint_sync_time,

140 + buffers_checkpoint_bytes,

141 + buffers_clean_bytes,

142 + maxwritten_clean,

143 + buffers_backend_bytes,

144 + buffers_backend_fsync,

145 + buffers_alloc_bytes

146 +`

147 + rows := sqlmock.NewRows([]string{

148 + "checkpoints_timed",

149 + "checkpoints_req",

150 + "checkpoint_write_time",

151 + "checkpoint_sync_time",

152 + "buffers_checkpoint_bytes",

153 + "buffers_clean_bytes",

154 + "maxwritten_clean",

155 + "buffers_backend_bytes",

156 + "buffers_backend_fsync",

157 + "buffers_alloc_bytes",

158 + }).AddRow(

159 + "1814",

160 + "16",

161 + "167",

162 + "47",

163 + "32768",

164 + "0",

165 + "0",

166 + "0",

167 + "0",

168 + "27295744",

169 + )

170 +

171 + mock.ExpectQuery(query).WillReturnRows(rows).RowsWillBeClosed()

172 +

173 + coll.Driver = "pgx"

174 + coll.DSN = "postgres://user:pass@localhost/db"

175 + coll.Metrics = []ConfigMetricBlock{

176 + {

177 + ID: "bgwriter",

178 + Mode: "columns",

179 + Query: query,

180 + Charts: []ConfigChartConfig{

181 + {

182 + Title: "bgwriter",

183 + Context: "pg.bgwriter",

184 + Family: "bgwriter",

185 + Units: "bytes",

186 + Dims: []ConfigDimConfig{

187 + {Name: "checkpoints_timed", Source: "checkpoints_timed"},

188 + {Name: "checkpoints_req", Source: "checkpoints_req"},

189 + {Name: "checkpoint_write_time", Source: "checkpoint_write_time"},

190 + {Name: "checkpoint_sync_time", Source: "checkpoint_sync_time"},

191 + {Name: "buffers_checkpoint_bytes", Source: "buffers_checkpoint_bytes"},

192 + {Name: "buffers_clean_bytes", Source: "buffers_clean_bytes"},

193 + {Name: "maxwritten_clean", Source: "maxwritten_clean"},

194 + {Name: "buffers_backend_bytes", Source: "buffers_backend_bytes"},

195 + {Name: "buffers_backend_fsync", Source: "buffers_backend_fsync"},

196 + {Name: "buffers_alloc_bytes", Source: "buffers_alloc_bytes"},

197 + },

198 + },

199 + },

200 + },

201 + }

202 + },

203 + check: func(t *testing.T, mx map[string]int64) {

204 + chartID := "pgx_bgwriter_pg.bgwriter"

205 +

206 + expected := map[string]int64{

207 + buildDimID(chartID, "checkpoints_timed"): 1814,

208 + buildDimID(chartID, "checkpoints_req"): 16,

209 + buildDimID(chartID, "checkpoint_write_time"): 167,

210 + buildDimID(chartID, "checkpoint_sync_time"): 47,

211 + buildDimID(chartID, "buffers_checkpoint_bytes"): 32768,

212 + buildDimID(chartID, "buffers_clean_bytes"): 0,

213 + buildDimID(chartID, "maxwritten_clean"): 0,

214 + buildDimID(chartID, "buffers_backend_bytes"): 0,

215 + buildDimID(chartID, "buffers_backend_fsync"): 0,

216 + buildDimID(chartID, "buffers_alloc_bytes"): 27295744,

217 + }

218 +

219 + for k, want := range expected {

220 + got, ok := mx[k]

221 + require.True(t, ok, "expected metric %s", k)

222 + assert.EqualValues(t, want, got, "metric %s", k)

223 + }

224 + },

225 + },

226 +

227 + "columns: metrics as columns with label (datname)": {

228 + prepare: func(t *testing.T, mock sqlmock.Sqlmock, coll *Collector) {

229 + query := `

230 +SELECT

231 + datname,

232 + confl_tablespace,

233 + confl_lock,

234 + confl_snapshot,

235 + confl_bufferpin,

236 + confl_deadlock

237 +`

238 + rows := sqlmock.NewRows([]string{

239 + "datname",

240 + "confl_tablespace",

241 + "confl_lock",

242 + "confl_snapshot",

243 + "confl_bufferpin",

244 + "confl_deadlock",

245 + }).

246 + AddRow("postgres", "0", "0", "0", "0", "0").

247 + AddRow("production", "0", "0", "0", "0", "0")

248 +

249 + mock.ExpectQuery(query).WillReturnRows(rows).RowsWillBeClosed()

250 +

251 + coll.Driver = "pgx"

252 + coll.DSN = "postgres://user:pass@localhost/db"

253 + coll.Metrics = []ConfigMetricBlock{

254 + {

255 + ID: "conflicts",

256 + Mode: "columns",

257 + Query: query,

258 + LabelsFromRow: []ConfigLabelFromRow{

259 + {Source: "datname", Name: "db"},

260 + },

261 + Charts: []ConfigChartConfig{

262 + {

263 + Title: "conflicts",

264 + Context: "pg.conflicts",

265 + Family: "conflicts",

266 + Units: "conflicts",

267 + Dims: []ConfigDimConfig{

268 + {Name: "confl_tablespace", Source: "confl_tablespace"},

269 + {Name: "confl_lock", Source: "confl_lock"},

270 + {Name: "confl_snapshot", Source: "confl_snapshot"},

271 + {Name: "confl_bufferpin", Source: "confl_bufferpin"},

272 + {Name: "confl_deadlock", Source: "confl_deadlock"},

273 + },

274 + },

275 + },

276 + },

277 + }

278 + },

279 + check: func(t *testing.T, mx map[string]int64) {

280 + chartPostgres := "pgx_conflicts_pg.conflicts_postgres"

281 + chartProduction := "pgx_conflicts_pg.conflicts_production"

282 +

283 + keys := []string{

284 + "confl_tablespace",

285 + "confl_lock",

286 + "confl_snapshot",

287 + "confl_bufferpin",

288 + "confl_deadlock",

289 + }

290 +

291 + for _, dim := range keys {

292 + k1 := buildDimID(chartPostgres, dim)

293 + k2 := buildDimID(chartProduction, dim)

294 +

295 + v1, ok1 := mx[k1]

296 + v2, ok2 := mx[k2]

297 +

298 + require.True(t, ok1, "expected metric %s", k1)

299 + require.True(t, ok2, "expected metric %s", k2)

300 +

301 + assert.EqualValues(t, 0, v1, "metric %s", k1)

302 + assert.EqualValues(t, 0, v2, "metric %s", k2)

303 + }

304 + },

305 + },

306 +

307 + "columns: single column table": {

308 + prepare: func(t *testing.T, mock sqlmock.Sqlmock, coll *Collector) {

309 + query := `SELECT extract`

310 + rows := sqlmock.NewRows([]string{"extract"}).

311 + AddRow("499906.075943")

312 +

313 + mock.ExpectQuery(query).WillReturnRows(rows).RowsWillBeClosed()

314 +

315 + coll.Driver = "pgx"

316 + coll.DSN = "postgres://user:pass@localhost/db"

317 + coll.Metrics = []ConfigMetricBlock{

318 + {

319 + ID: "uptime",

320 + Mode: "columns",

321 + Query: query,

322 + Charts: []ConfigChartConfig{

323 + {

324 + Title: "uptime",

325 + Context: "pg.uptime",

326 + Family: "uptime",

327 + Units: "seconds",

328 + Dims: []ConfigDimConfig{

329 + {Name: "extract", Source: "extract"},

330 + },

331 + },

332 + },

333 + },

334 + }

335 + },

336 + check: func(t *testing.T, mx map[string]int64) {

337 + chartID := "pgx_uptime_pg.uptime"

338 + key := buildDimID(chartID, "extract")

339 +

340 + got, ok := mx[key]

341 + require.True(t, ok, "expected metric %s", key)

342 + // 499906.075943 -> 499906 after truncation

343 + assert.EqualValues(t, 499906, got)

344 + },

345 + },

346 +

347 + "kv: metric names as row keys (state -> count)": {

348 + prepare: func(t *testing.T, mock sqlmock.Sqlmock, coll *Collector) {

349 + query := `

350 +SELECT

351 + state,

352 + count

353 +`

354 + rows := sqlmock.NewRows([]string{"state", "count"}).

355 + AddRow("active", "1").

356 + AddRow("idle", "14").

357 + AddRow("idle in transaction", "7").

358 + AddRow("idle in transaction (aborted)", "1").

359 + AddRow("fastpath function call", "1").

360 + AddRow("disabled", "1")

361 +

362 + mock.ExpectQuery(query).WillReturnRows(rows).RowsWillBeClosed()

363 +

364 + coll.Driver = "pgx"

365 + coll.DSN = "postgres://user:pass@localhost/db"

366 + coll.Metrics = []ConfigMetricBlock{

367 + {

368 + ID: "activity_states",

369 + Mode: "kv",

370 + Query: query,

371 + KVMode: &ConfigKVMode{

372 + NameCol: "state",

373 + ValueCol: "count",

374 + },

375 + Charts: []ConfigChartConfig{

376 + {

377 + Title: "activity states",

378 + Context: "pg.activity_states",

379 + Family: "activity states",

380 + Units: "events",

381 + Dims: []ConfigDimConfig{

382 + {Name: "active", Source: "active"},

383 + {Name: "idle", Source: "idle"},

384 + {Name: "idle_in_transaction", Source: "idle in transaction"},

385 + {Name: "idle_in_transaction_aborted", Source: "idle in transaction (aborted)"},

386 + {Name: "fastpath_function_call", Source: "fastpath function call"},

387 + {Name: "disabled", Source: "disabled"},

388 + },

389 + },

390 + },

391 + },

392 + }

393 + },

394 + check: func(t *testing.T, mx map[string]int64) {

395 + chartID := "pgx_activity_states_pg.activity_states"

396 +

397 + expected := map[string]int64{

398 + buildDimID(chartID, "active"): 1,

399 + buildDimID(chartID, "idle"): 14,

400 + buildDimID(chartID, "idle_in_transaction"): 7,

401 + buildDimID(chartID, "idle_in_transaction_aborted"): 1,

402 + buildDimID(chartID, "fastpath_function_call"): 1,

403 + buildDimID(chartID, "disabled"): 1,

404 + }

405 +

406 + for k, want := range expected {

407 + got, ok := mx[k]

408 + require.True(t, ok, "expected metric %s", k)

409 + assert.EqualValues(t, want, got, "metric %s", k)

410 + }

411 + },

412 + },

413 +

414 + "columns: state-type metric with label from state": {

415 + prepare: func(t *testing.T, mock sqlmock.Sqlmock, coll *Collector) {

416 + query := `

417 +SELECT

418 + datname,

419 + state,

420 + xact_running_time,

421 + query_running_time

422 +`

423 + rows := sqlmock.NewRows([]string{

424 + "datname",

425 + "state",

426 + "xact_running_time",

427 + "query_running_time",

428 + }).

429 + AddRow("some_db", "idle in transaction", "574.530219", "574.315061").

430 + AddRow("some_db", "idle in transaction", "574.867167", "574.330322").

431 + AddRow("postgres", "active", "0.000000", "0.000000").

432 + AddRow("some_db", "idle in transaction", "574.807256", "574.377105").

433 + AddRow("some_db", "idle in transaction", "574.680244", "574.357246").

434 + AddRow("some_db", "idle in transaction", "574.800283", "574.330328").

435 + AddRow("some_db", "idle in transaction", "574.396730", "574.290165").

436 + AddRow("some_db", "idle in transaction", "574.665428", "574.337164")

437 +

438 + mock.ExpectQuery(query).WillReturnRows(rows).RowsWillBeClosed()

439 +

440 + coll.Driver = "pgx"

441 + coll.DSN = "postgres://user:pass@localhost/db"

442 + coll.Metrics = []ConfigMetricBlock{

443 + {

444 + ID: "xact_state",

445 + Mode: "columns",

446 + Query: query,

447 + LabelsFromRow: []ConfigLabelFromRow{

448 + {Source: "state", Name: "state"},

449 + },

450 + Charts: []ConfigChartConfig{

451 + {

452 + Title: "activity",

453 + Context: "pg.activity",

454 + Family: "activity",

455 + Units: "seconds",

456 + Dims: []ConfigDimConfig{

457 + {Name: "xact_running_time", Source: "xact_running_time"},

458 + {Name: "query_running_time", Source: "query_running_time"},

459 + },

460 + },

461 + },

462 + },

463 + }

464 + },

465 + check: func(t *testing.T, mx map[string]int64) {

466 + // Chart instances are split by state label.

467 + idleChartID := "pgx_xact_state_pg.activity_idle in transaction"

468 + activeChartID := "pgx_xact_state_pg.activity_active"

469 +

470 + xactIdleKey := buildDimID(idleChartID, "xact_running_time")

471 + queryIdleKey := buildDimID(idleChartID, "query_running_time")

472 + xactActiveKey := buildDimID(activeChartID, "xact_running_time")

473 + queryActiveKey := buildDimID(activeChartID, "query_running_time")

474 +

475 + // idle in transaction: aggregated sum over multiple rows -> > 0

476 + xIdle, ok := mx[xactIdleKey]

477 + require.True(t, ok, "expected metric %s", xactIdleKey)

478 + assert.Greater(t, xIdle, int64(0))

479 +

480 + qIdle, ok := mx[queryIdleKey]

481 + require.True(t, ok, "expected metric %s", queryIdleKey)

482 + assert.Greater(t, qIdle, int64(0))

483 +

484 + // active row has 0 durations.

485 + xAct, ok := mx[xactActiveKey]

486 + require.True(t, ok, "expected metric %s", xactActiveKey)

487 + assert.EqualValues(t, 0, xAct)

488 +

489 + qAct, ok := mx[queryActiveKey]

490 + require.True(t, ok, "expected metric %s", queryActiveKey)

491 + assert.EqualValues(t, 0, qAct)

492 + },

493 + },

494 +

495 + "columns: state-type metric with state mapped to metrics": {

496 + prepare: func(t *testing.T, mock sqlmock.Sqlmock, coll *Collector) {

497 + query := `

498 +SELECT

499 + datname,

500 + state,

501 + xact_running_time,

502 + query_running_time

503 +`

504 + rows := sqlmock.NewRows([]string{

505 + "datname",

506 + "state",

507 + "xact_running_time",

508 + "query_running_time",

509 + }).

510 + AddRow("some_db", "idle in transaction", "574.530219", "574.315061").

511 + AddRow("some_db", "idle in transaction", "574.867167", "574.330322").

512 + AddRow("postgres", "active", "0.000000", "0.000000").

513 + AddRow("some_db", "idle in transaction", "574.807256", "574.377105").

514 + AddRow("some_db", "idle in transaction", "574.680244", "574.357246").

515 + AddRow("some_db", "idle in transaction", "574.800283", "574.330328").

516 + AddRow("some_db", "idle in transaction", "574.396730", "574.290165").

517 + AddRow("some_db", "idle in transaction", "574.665428", "574.337164")

518 +

519 + mock.ExpectQuery(query).WillReturnRows(rows).RowsWillBeClosed()

520 +

521 + coll.Driver = "pgx"

522 + coll.DSN = "postgres://user:pass@localhost/db"

523 + coll.Metrics = []ConfigMetricBlock{

524 + {

525 + ID: "xact_state_map",

526 + Mode: "columns",

527 + Query: query,

528 + Charts: []ConfigChartConfig{

529 + {

530 + Title: "activity",

531 + Context: "pg.activity",

532 + Family: "activity",

533 + Units: "status",

534 + Dims: []ConfigDimConfig{

535 + {

536 + Name: "state_active",

537 + Source: "state",

538 + StatusWhen: &ConfigStatusWhen{

539 + Equals: "active",

540 + },

541 + },

542 + {

543 + Name: "state_idle_in_transaction",

544 + Source: "state",

545 + StatusWhen: &ConfigStatusWhen{

546 + Equals: "idle in transaction",

547 + },

548 + },

549 + },

550 + },

551 + },

552 + },

553 + }

554 + },

555 + check: func(t *testing.T, mx map[string]int64) {

556 + chartID := "pgx_xact_state_map_pg.activity"

557 +

558 + activeKey := buildDimID(chartID, "state_active")

559 + idleKey := buildDimID(chartID, "state_idle_in_transaction")

560 +

561 + active, ok := mx[activeKey]

562 + require.True(t, ok, "expected metric %s", activeKey)

563 + assert.EqualValues(t, 1, active)

564 +

565 + idle, ok := mx[idleKey]

566 + require.True(t, ok, "expected metric %s", idleKey)

567 + assert.EqualValues(t, 1, idle)

568 + },

569 + },

570 + }

571 +

572 + for name, tt := range tests {

573 + t.Run(name, func(t *testing.T) {

574 + db, mock, err := sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual))

575 + require.NoError(t, err)

576 + defer func() { _ = db.Close() }()

577 +

578 + coll := New()

579 + coll.db = db

580 +

581 + tt.prepare(t, mock, coll)

582 +

583 + require.NoError(t, coll.Init(context.Background()))

584 +

585 + mx := coll.Collect(context.Background())

586 + require.NotNil(t, mx)

587 +

588 + tt.check(t, mx)

589 +

590 + assert.NoError(t, mock.ExpectationsWereMet())

591 + })

592 + }

593 +}

594 +

595 +func TestRedactDSN(t *testing.T) {

596 + tests := map[string]struct {

597 + input string

598 + expected string

599 + }{

600 + "simple user:pass@host": {

601 + input: "user:password@localhost",

602 + expected: "user:****@localhost",

603 + },

604 + "simple user:pass@host:port": {

605 + input: "user:password@localhost:5432",

606 + expected: "user:****@localhost:5432",

607 + },

608 + "no credentials just host": {

609 + input: "localhost:5432",

610 + expected: "localhost:5432",

611 + },

612 + "URL without credentials": {

613 + input: "postgresql://localhost/dbname",

614 + expected: "postgresql://localhost/dbname",

615 + },

616 + "colon in host but no password": {

617 + input: "localhost:5432/db",

618 + expected: "localhost:5432/db",

619 + },

620 + "empty string": {

621 + input: "",

622 + expected: "",

623 + },

624 + "just scheme": {

625 + input: "postgresql://",

626 + expected: "postgresql://",

627 + },

628 + "simple user@host (no password)": {

629 + input: "user@localhost",

630 + expected: "****@localhost",

631 + },

632 + "postgresql URL with password": {

633 + input: "postgresql://user:password@localhost/dbname",

634 + expected: "postgresql://user:****@localhost/dbname",

635 + },

636 + "postgresql URL with password and port": {

637 + input: "postgresql://user:password@localhost:5432/dbname",

638 + expected: "postgresql://user:****@localhost:5432/dbname",

639 + },

640 + "postgresql URL without password": {

641 + input: "postgresql://user@localhost/dbname",

642 + expected: "postgresql://****@localhost/dbname",

643 + },

644 + "mysql URL with password": {

645 + input: "mysql://root:secret@localhost:3306/mydb",

646 + expected: "mysql://root:****@localhost:3306/mydb",

647 + },

648 + "postgres URL with complex password": {

649 + input: "postgres://admin:p@ss:w0rd@localhost/db",

650 + expected: "postgres://admin:****@localhost/db",

651 + },

652 + "URL with query params": {

653 + input: "postgresql://user:pass@localhost/db?sslmode=disable",

654 + expected: "postgresql://user:****@localhost/db?sslmode=disable",

655 + },

656 + "user with special chars in password": {

657 + input: "user:p@ssw0rd!@localhost",

658 + expected: "user:****@localhost",

659 + },

660 + "redis URL": {

661 + input: "redis://user:password@localhost:6379/0",

662 + expected: "redis://user:****@localhost:6379/0",

663 + },

664 + "mongodb URL": {

665 + input: "mongodb://admin:secret@localhost:27017/mydb",

666 + expected: "mongodb://admin:****@localhost:27017/mydb",

667 + },

668 + "URL with IP address": {

669 + input: "postgresql://user:pass@192.168.1.1:5432/db",

670 + expected: "postgresql://user:****@192.168.1.1:5432/db",

671 + },

672 + }

673 +

674 + for name, tc := range tests {

675 + t.Run(name, func(t *testing.T) {

676 + got := redactDSN(tc.input)

677 +

678 + assert.Equal(t, tc.expected, got)

679 + })

680 + }

681 +}

src/go/plugin/go.d/collector/sql/config.go new

+313

@@ -0,0 +1,313 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +package sql

4 +

5 +import (

6 + "errors"

7 + "fmt"

8 + "regexp"

9 + "strings"

10 +

11 + "github.com/netdata/netdata/go/plugins/pkg/confopt"

12 +)

13 +

14 +type Config struct {

15 + UpdateEvery int `yaml:"update_every,omitempty" json:"update_every"`

16 + AutoDetectionRetry int `yaml:"autodetection_retry,omitempty" json:"autodetection_retry"`

17 +

18 + Driver string `yaml:"driver" json:"driver"`

19 + DSN string `yaml:"dsn" json:"dsn"`

20 + Timeout confopt.Duration `yaml:"timeout" json:"timeout"`

21 +

22 + StaticLabels map[string]string `yaml:"static_labels,omitempty" json:"static_labels"`

23 + Queries []ConfigQueryDef `yaml:"queries,omitempty" json:"queries"`

24 + Metrics []ConfigMetricBlock `yaml:"metrics,omitempty" json:"metrics"`

25 +}

26 +

27 +type (

28 + ConfigQueryDef struct {

29 + ID string `yaml:"id" json:"id"`

30 + Query string `yaml:"query" json:"query"`

31 + }

32 +

33 + ConfigMetricBlock struct {

34 + ID string `yaml:"id" json:"id"`

35 + QueryRef string `yaml:"query_ref,omitempty" json:"query_ref"`

36 + Query string `yaml:"query,omitempty" json:"query"`

37 + Mode string `yaml:"mode" json:"mode"` // "columns" | "kv"

38 + KVMode *ConfigKVMode `yaml:"kv_mode,omitempty" json:"kv_mode"`

39 + LabelsFromRow []ConfigLabelFromRow `yaml:"labels_from_row,omitempty" json:"labels_from_row"`

40 + Charts []ConfigChartConfig `yaml:"charts,omitempty" json:"charts"`

41 + }

42 + ConfigKVMode struct {

43 + NameCol string `yaml:"name_col,omitempty" json:"name_col"`

44 + ValueCol string `yaml:"value_col,omitempty" json:"value_col"`

45 + }

46 + ConfigLabelFromRow struct {

47 + Source string `yaml:"source" json:"source"`

48 + Name string `yaml:"name" json:"name"`

49 + //ValueMap map[string]string `yaml:"value_map" json:"value_map"`

50 + }

51 +)

52 +

53 +type (

54 + ConfigChartConfig struct {

55 + Title string `yaml:"title" json:"title"`

56 + Context string `yaml:"context" json:"context"`

57 + Family string `yaml:"family" json:"family"`

58 + Type string `yaml:"type" json:"type"`

59 + Units string `yaml:"units" json:"units"`

60 + Algorithm string `yaml:"algorithm" json:"algorithm"`

61 + Dims []ConfigDimConfig `yaml:"dims" json:"dims"`

62 + }

63 + ConfigDimConfig struct {

64 + Name string `yaml:"name" json:"name"`

65 + Source string `yaml:"source" json:"source"`

66 + StatusWhen *ConfigStatusWhen `yaml:"status_when,omitempty" json:"status_when,omitempty"`

67 + }

68 + ConfigStatusWhen struct {

69 + Equals string `yaml:"equals,omitempty" json:"equals,omitempty"`

70 + In []string `yaml:"in,omitempty" json:"in,omitempty"`

71 + Match string `yaml:"match,omitempty" json:"match,omitempty"`

72 +

73 + re *regexp.Regexp

74 + }

75 +)

76 +

77 +func (c *Collector) validateConfig() error {

78 + var errs []error

79 +

80 + if c.Driver == "" {

81 + errs = append(errs, errors.New("driver required"))

82 + } else if !supportedDrivers[c.Driver] {

83 + errs = append(errs, fmt.Errorf("unsupported driver %q", c.Driver))

84 + }

85 + if c.DSN == "" {

86 + errs = append(errs, errors.New("dsn required"))

87 + }

88 +

89 + if len(c.Metrics) == 0 {

90 + errs = append(errs, errors.New("missing metrics"))

91 + }

92 +

93 + queryIdx := map[string]bool{}

94 + for i := range c.Queries {

95 + errs = append(errs, c.Queries[i].validate(i, queryIdx)...)

96 + }

97 +

98 + for i := range c.Metrics {

99 + errs = append(errs, c.Metrics[i].validate(i, queryIdx)...)

100 + }

101 +

102 + return errors.Join(errs...)

103 +}

104 +

105 +// ---- Per-struct validation helpers ----

106 +

107 +func (q *ConfigQueryDef) validate(idx int, seen map[string]bool) []error {

108 + var errs []error

109 + qidx := idx + 1

110 +

111 + if q.ID == "" {

112 + errs = append(errs, fmt.Errorf("queries[%d] missing id", qidx))

113 + }

114 + if q.Query == "" {

115 + errs = append(errs, fmt.Errorf("queries[%d] missing query", qidx))

116 + }

117 +

118 + if q.ID != "" {

119 + if _, dup := seen[q.ID]; dup {

120 + errs = append(errs, fmt.Errorf("queries[%d] duplicate id %q", qidx, q.ID))

121 + }

122 + seen[q.ID] = true

123 + }

124 +

125 + return errs

126 +}

127 +

128 +func (m *ConfigMetricBlock) validate(idx int, queryIdx map[string]bool) []error {

129 + var errs []error

130 + midx := idx + 1

131 +

132 + if m.ID == "" {

133 + errs = append(errs, fmt.Errorf("metrics[%d] missing id", midx))

134 + }

135 +

136 + hasRef := strings.TrimSpace(m.QueryRef) != ""

137 + hasInline := strings.TrimSpace(m.Query) != ""

138 + switch {

139 + case hasRef && hasInline:

140 + errs = append(errs, fmt.Errorf("metrics[%d] must set exactly one of query_ref or query, not both", midx))

141 + case !hasRef && !hasInline:

142 + errs = append(errs, fmt.Errorf("metrics[%d] must set exactly one of query_ref or query", midx))

143 + case hasRef:

144 + if _, ok := queryIdx[m.QueryRef]; !ok {

145 + errs = append(errs, fmt.Errorf("metrics[%d] query_ref %q not found in queries", midx, m.QueryRef))

146 + }

147 + }

148 +

149 + mode := strings.ToLower(strings.TrimSpace(m.Mode))

150 + switch mode {

151 + case "kv":

152 + if m.KVMode == nil {

153 + errs = append(errs, fmt.Errorf("metrics[%d] kv mode requires kv_mode to be defined", midx))

154 + } else {

155 + if strings.TrimSpace(m.KVMode.NameCol) == "" {

156 + errs = append(errs, fmt.Errorf("metrics[%d] kv_mode.name_col is required in kv mode", midx))

157 + }

158 + if strings.TrimSpace(m.KVMode.ValueCol) == "" {

159 + errs = append(errs, fmt.Errorf("metrics[%d] kv_mode.value_col is required in kv mode", midx))

160 + }

161 + if strings.EqualFold(m.KVMode.NameCol, m.KVMode.ValueCol) {

162 + errs = append(errs, fmt.Errorf("metrics[%d] kv_mode.name_col must differ from kv_mode.value_col", midx))

163 + }

164 + }

165 + case "columns", "":

166 + default:

167 + errs = append(errs, fmt.Errorf("metrics[%d] invalid mode %q (expected: columns|kv)", midx, m.Mode))

168 + }

169 +

170 + labelCols := map[string]bool{}

171 + for j := range m.LabelsFromRow {

172 + errs = append(errs, m.LabelsFromRow[j].validate(midx, j, labelCols)...)

173 + }

174 +

175 + if len(m.Charts) == 0 {

176 + errs = append(errs, fmt.Errorf("metrics[%d].%s missing charts", midx, m.ID))

177 + }

178 +

179 + for k := range m.Charts {

180 + errs = append(errs, m.Charts[k].validate(idx, m.ID, k, mode, labelCols, m.KVMode)...)

181 + }

182 +

183 + return errs

184 +}

185 +

186 +func (lf *ConfigLabelFromRow) validate(metricIdx, lfIdx int, labelCols map[string]bool) []error {

187 + var errs []error

188 + midx := metricIdx + 1

189 + lidx := lfIdx + 1

190 +

191 + if lf.Source == "" {

192 + errs = append(errs, fmt.Errorf("metrics[%d].labels_from_row[%d] missing source", midx, lidx))

193 + }

194 + if lf.Name == "" {

195 + errs = append(errs, fmt.Errorf("metrics[%d].labels_from_row[%d] missing name", midx, lidx))

196 + }

197 + if lf.Source != "" {

198 + labelCols[strings.ToLower(lf.Source)] = true

199 + }

200 +

201 + return errs

202 +}

203 +

204 +func (ch *ConfigChartConfig) validate(metricIdx int, metricID string, chartIdx int, mode string, labelCols map[string]bool, kv *ConfigKVMode) []error {

205 + var errs []error

206 + midx := metricIdx + 1

207 + cidx := chartIdx + 1

208 +

209 + if strings.TrimSpace(ch.Context) == "" {

210 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d] missing context", midx, cidx))

211 + }

212 + if strings.TrimSpace(ch.Title) == "" {

213 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d] missing title", midx, cidx))

214 + }

215 + if strings.TrimSpace(ch.Family) == "" {

216 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d] missing family", midx, cidx))

217 + }

218 + if strings.TrimSpace(ch.Units) == "" {

219 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d] missing units", midx, cidx))

220 + }

221 +

222 + seenDims := map[string]bool{}

223 + for d := range ch.Dims {

224 + errs = append(errs, ch.Dims[d].validate(metricIdx, metricID, chartIdx, d, mode, labelCols, kv, seenDims)...)

225 + }

226 +

227 + return errs

228 +}

229 +

230 +func (dm *ConfigDimConfig) validate(

231 + metricIdx int,

232 + metricID string,

233 + chartIdx int,

234 + dimIdx int,

235 + mode string,

236 + labelCols map[string]bool,

237 + kv *ConfigKVMode,

238 + seenDims map[string]bool,

239 +) []error {

240 + var errs []error

241 + midx := metricIdx + 1

242 + cidx := chartIdx + 1

243 + didx := dimIdx + 1

244 +

245 + if strings.TrimSpace(dm.Name) == "" {

246 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d].dims[%d] missing name", midx, cidx, didx))

247 + }

248 + if strings.TrimSpace(dm.Source) == "" {

249 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d].dims[%d] missing source", midx, cidx, didx))

250 + }

251 +

252 + // unique dim names (case-insensitive)

253 + if dm.Name != "" {

254 + key := strings.ToLower(dm.Name)

255 + if _, ok := seenDims[key]; ok {

256 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d] duplicate dim name %q", midx, cidx, dm.Name))

257 + } else {

258 + seenDims[key] = true

259 + }

260 + }

261 +

262 + if dm.StatusWhen != nil {

263 + errs = append(errs, dm.StatusWhen.validate(metricIdx, metricID, chartIdx, dimIdx)...)

264 + }

265 +

266 + switch mode {

267 + case "columns":

268 + if _, clash := labelCols[strings.ToLower(dm.Source)]; clash {

269 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d].dims[%d] source %q conflicts with labels_from_row source", midx, cidx, didx, dm.Source))

270 + }

271 + case "kv":

272 + // In kv mode, dim.source must be a KEY name (not a column).

273 + if kv != nil && strings.EqualFold(dm.Source, kv.ValueCol) {

274 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d].dims[%d] source %q equals kv_mode.value_col; dim.source must be a KEY name", midx, cidx, didx, dm.Source))

275 + }

276 + }

277 +

278 + return errs

279 +}

280 +

281 +func (sw *ConfigStatusWhen) validate(metricIdx int, metricID string, chartIdx, dimIdx int) []error {

282 + var errs []error

283 + midx := metricIdx + 1

284 + cidx := chartIdx + 1

285 + didx := dimIdx + 1

286 +

287 + count := 0

288 + if sw.Equals != "" {

289 + count++

290 + }

291 + if len(sw.In) > 0 {

292 + count++

293 + }

294 + if strings.TrimSpace(sw.Match) != "" {

295 + re, err := regexp.Compile(sw.Match)

296 + if err != nil {

297 + errs = append(errs, fmt.Errorf("invalid regex in status_when.match for metric %q: %w", metricID, err))

298 + } else {

299 + // store compiled regex for later use

300 + sw.re = re

301 + }

302 + count++

303 + }

304 +

305 + if count == 0 {

306 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d].dims[%d].status_when must have exactly one of equals|in|match", midx, cidx, didx))

307 + }

308 + if count > 1 {

309 + errs = append(errs, fmt.Errorf("metrics[%d].charts[%d].dims[%d].status_when must not set multiple selectors", midx, cidx, didx))

310 + }

311 +

312 + return errs

313 +}

src/go/plugin/go.d/collector/sql/config_schema.json new

+401

@@ -0,0 +1,401 @@

1 +{

2 + "jsonSchema": {

3 + "$schema": "http://json-schema.org/draft-07/schema#",

4 + "title": "SQL collector configuration.",

5 + "type": "object",

6 + "properties": {

7 + "update_every": {

8 + "title": "Update every",

9 + "description": "How often to collect metrics from the database, in seconds.",

10 + "type": "integer",

11 + "minimum": 1,

12 + "default": 1

13 + },

14 + "autodetection_retry": {

15 + "title": "Detection retry",

16 + "description": "How often Netdata retries failed automatic detection jobs, in seconds. Set to 0 to disable retries.",

17 + "type": "integer",

18 + "minimum": 0,

19 + "default": 0

20 + },

21 + "driver": {

22 + "title": "Driver",

23 + "description": "SQL driver / database engine to use. Supported values: `mysql` (MySQL/MariaDB), `pgx` (PostgreSQL), `oracle` (OracleDB), `sqlserver` (Microsoft SQL Server).",

24 + "type": "string",

25 + "enum": [

26 + "mysql",

27 + "pgx",

28 + "oracle",

29 + "sqlserver"

30 + ],

31 + "default": "mysql"

32 + },

33 + "dsn": {

34 + "title": "DSN",

35 + "description": "Database connection string (DSN). The format depends on the selected driver.",

36 + "type": "string"

37 + },

38 + "timeout": {

39 + "title": "Timeout",

40 + "description": "Maximum time allowed for a single query and connection check, in seconds.",

41 + "type": "number",

42 + "minimum": 0,

43 + "default": 5

44 + },

45 + "static_labels": {

46 + "title": "Static labels",

47 + "description": "Key/value labels automatically added to every chart created by this job (for example `env=prod`, `region=eu-west`).",

48 + "type": "object",

49 + "additionalProperties": {

50 + "type": "string"

51 + }

52 + },

53 + "queries": {

54 + "title": "Reusable queries",

55 + "description": "Optional list of reusable SQL queries that metric blocks can reference via `query_ref` to avoid repeating SQL text.",

56 + "type": "array",

57 + "items": {

58 + "title": "Query",

59 + "type": "object",

60 + "properties": {

61 + "id": {

62 + "title": "ID",

63 + "description": "Unique query identifier referenced from metric blocks using `query_ref` within this job.",

64 + "type": "string"

65 + },

66 + "query": {

67 + "title": "Query",

68 + "description": "SQL statement text executed when this query is referenced.",

69 + "type": "string"

70 + }

71 + },

72 + "required": [

73 + "id",

74 + "query"

75 + ]

76 + }

77 + },

78 + "metrics": {

79 + "title": "Metric blocks",

80 + "description": "List of metric blocks. Each block runs one query and maps its result set into one or more Netdata charts.",

81 + "type": "array",

82 + "items": {

83 + "title": "Metric block",

84 + "type": "object",

85 + "properties": {

86 + "id": {

87 + "title": "ID",

88 + "description": "Metric block identifier. Must be unique within this job.",

89 + "type": "string"

90 + },

91 + "query_ref": {

92 + "title": "Query ref",

93 + "description": "Reference to a reusable query defined in `queries`. Exactly one of `query_ref` or `query` must be set.",

94 + "type": "string"

95 + },

96 + "query": {

97 + "title": "Inline query",

98 + "description": "Inline SQL statement for this metric block. Exactly one of `query_ref` or `query` must be set.",

99 + "type": "string"

100 + },

101 + "mode": {

102 + "title": "Mode",

103 + "description": "How to interpret the result rows. `columns` treats selected columns as metrics. `kv` treats one column as metric name (key) and another as its value.",

104 + "type": "string",

105 + "enum": [

106 + "columns",

107 + "kv"

108 + ],

109 + "default": "columns"

110 + },

111 + "labels_from_row": {

112 + "title": "Labels from row",

113 + "description": "Optional labels derived from result columns. Each unique combination of label values produces a separate chart instance.",

114 + "type": "array",

115 + "items": {

116 + "title": "Label from row",

117 + "type": "object",

118 + "properties": {

119 + "source": {

120 + "title": "Source column",

121 + "description": "Name of the result column whose value will be used as the label value.",

122 + "type": "string"

123 + },

124 + "name": {

125 + "title": "Label key",

126 + "description": "Label key/name added to the chart (for example `db`, `schema`, `table`).",

127 + "type": "string"

128 + }

129 + },

130 + "required": [

131 + "source",

132 + "name"

133 + ]

134 + }

135 + },

136 + "charts": {

137 + "title": "Charts",

138 + "description": "One or more Netdata charts generated from this metric block.",

139 + "type": "array",

140 + "items": {

141 + "title": "Chart",

142 + "type": "object",

143 + "properties": {

144 + "title": {

145 + "title": "Title",

146 + "description": "Human-friendly chart title shown in Netdata dashboards.",

147 + "type": "string"

148 + },

149 + "context": {

150 + "title": "Context",

151 + "description": "Base metric context name for the chart. The collector will prefix it as `sql.<driver>_<context>` to form the full Netdata context. Contexts define the metric namespace, control alert templates, and group charts across jobs and nodes.",

152 + "type": "string"

153 + },

154 + "family": {

155 + "title": "Family",

156 + "description": "Chart family used to organize charts in the Netdata UI. You can use '/' to create hierarchical grouping levels (e.g. `Database/Connections/Active`). Each level becomes a nested section in the dashboard.",

157 + "type": "string"

158 + },

159 + "type": {

160 + "title": "Type",

161 + "description": "Visual type of the chart.",

162 + "type": "string",

163 + "enum": [

164 + "line",

165 + "stacked",

166 + "area"

167 + ],

168 + "default": "line"

169 + },

170 + "units": {

171 + "title": "Units",

172 + "description": "Unit label for all dimensions in this chart (for example `queries/s`, `bytes`, `connections`).",

173 + "type": "string"

174 + },

175 + "algorithm": {

176 + "title": "Algorithm",

177 + "description": "How Netdata interprets values: `absolute` for values that are already final, `incremental` for monotonically increasing counters.",

178 + "type": "string",

179 + "enum": [

180 + "absolute",

181 + "incremental"

182 + ],

183 + "default": "absolute"

184 + },

185 + "dims": {

186 + "title": "Dimensions",

187 + "description": "Definitions of the metrics (dimensions) that belong to this chart.",

188 + "type": "array",

189 + "items": {

190 + "title": "Dimension",

191 + "type": "object",

192 + "properties": {

193 + "name": {

194 + "title": "Name",

195 + "description": "Dimension identifier, unique within this chart. Also used as the legend name.",

196 + "type": "string"

197 + },

198 + "source": {

199 + "title": "Source",

200 + "description": "In `columns` mode: the numeric result column to read. In `kv` mode: the key name that must match a value in `kv_mode.name_col`.",

201 + "type": "string"

202 + },

203 + "status_when": {

204 + "title": "Status when",

205 + "description": "Optional condition that turns this dimension into a boolean 0/1 status metric. The dimension is 1 if any row matches the condition, otherwise 0.",

206 + "type": "object",

207 + "properties": {

208 + "equals": {

209 + "title": "Equals",

210 + "description": "Use 1 when the value is exactly equal to this literal, otherwise 0. Only one of `equals`, `in`, or `match` may be set.",

211 + "type": "string"

212 + },

213 + "in": {

214 + "title": "In",

215 + "description": "Use 1 when the value is equal to any element in this list, otherwise 0. Only one of `equals`, `in`, or `match` may be set.",

216 + "type": "array",

217 + "items": {

218 + "type": "string"

219 + }

220 + },

221 + "match": {

222 + "title": "Match (regex)",

223 + "description": "Use 1 when the value matches this regular expression, otherwise 0. Only one of `equals`, `in`, or `match` may be set.",

224 + "type": "string"

225 + }

226 + }

227 + }

228 + },

229 + "required": [

230 + "name",

231 + "source"

232 + ]

233 + }

234 + }

235 + },

236 + "required": [

237 + "title",

238 + "context",

239 + "family",

240 + "units",

241 + "dims"

242 + ]

243 + }

244 + }

245 + },

246 + "required": [

247 + "id",

248 + "mode",

249 + "charts"

250 + ],

251 + "dependencies": {

252 + "mode": {

253 + "oneOf": [

254 + {

255 + "properties": {

256 + "mode": {

257 + "const": "columns"

258 + }

259 + }

260 + },

261 + {

262 + "properties": {

263 + "mode": {

264 + "const": "kv"

265 + },

266 + "kv_mode": {

267 + "title": "KV mode",

268 + "description": "Configuration for `kv` mode that tells the collector which columns hold the key and value.",

269 + "type": "object",

270 + "properties": {

271 + "name_col": {

272 + "title": "Name column",

273 + "description": "Name of the result column that contains metric keys (for example `state`).",

274 + "type": "string"

275 + },

276 + "value_col": {

277 + "title": "Value column",

278 + "description": "Name of the result column that contains numeric metric values for each key.",

279 + "type": "string"

280 + }

281 + }

282 + }

283 + },

284 + "required": [

285 + "kv_mode"

286 + ]

287 + }

288 + ]

289 + }

290 + }

291 + }

292 + },

293 + "vnode": {

294 + "title": "Vnode",

295 + "description": "Name of the Virtual Node this job should send its metrics to. Leave empty to use the local node.",

296 + "type": "string"

297 + }

298 + },

299 + "required": [

300 + "driver",

301 + "dsn"

302 + ]

303 + },

304 + "uiSchema": {

305 + "uiOptions": {

306 + "fullPage": true

307 + },

308 + "ui:flavour": "tabs",

309 + "ui:options": {

310 + "tabs": [

311 + {

312 + "title": "Base",

313 + "fields": [

314 + "update_every",

315 + "autodetection_retry",

316 + "driver",

317 + "dsn",

318 + "timeout",

319 + "vnode"

320 + ]

321 + },

322 + {

323 + "title": "Metrics",

324 + "fields": [

325 + "metrics"

326 + ]

327 + },

328 + {

329 + "title": "Reusable Queries",

330 + "fields": [

331 + "queries"

332 + ]

333 + },

334 + {

335 + "title": "Static Labels",

336 + "fields": [

337 + "static_labels"

338 + ]

339 + }

340 + ]

341 + },

342 + "vnode": {

343 + "ui:placeholder": "To use this option, first create a Virtual Node and then reference its name here."

344 + },

345 + "autodetection_retry": {

346 + "ui:help": "This option determines how frequently (in seconds) Netdata will retry data collection jobs that failed initially; setting it to 0 disables this retry mechanism entirely."

347 + },

348 + "driver": {

349 + "ui:widget": "radio",

350 + "ui:options": {

351 + "inline": true

352 + }

353 + },

354 + "dsn": {

355 + "ui:placeholder": "Enter the database connection string",

356 + "ui:help": "**DSN format depends on the selected driver.**\n\n### MySQL / MariaDB\nFormat: `user:password@tcp(host:port)/dbname?param1=value1&paramN=valueN`\nExample:\n```\nroot:pass@tcp(127.0.0.1:3306)/\n```\n[MySQL DSN documentation](https://github.com/go-sql-driver/mysql#dsn-data-source-name)\n\n### PostgreSQL\nFormats: `postgresql://user:password@host:5432/dbname?param1=value1&paramN=valueN`\nExample:\n```\npostgresql://user:pass@localhost:5432/mydb\n```\n[PostgreSQL DSN documentation](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING-URIS)\n\n### Oracle\nFormat: `oracle://username:password@host:port/service?param1=value1&paramN=valueN`\nExample:\n```\noracle://scott:tiger@db.example.com:1521/ORCLPDB1?ssl=off\n```\n\n### SQL Server\nFormat: `sqlserver://username:password@host:port?database=dbname&param=value`\nExample:\n```\nsqlserver://sa:myStrong(!)Password@localhost:1433?database=master&connection+timeout=30\n```\n[SQL Server driver DSN documentation](https://github.com/denisenkom/go-mssqldb#connection-parameters-and-dsn)"

357 + },

358 + "static_labels": {

359 + "ui:help": "Optional key/value labels added to every chart created by this job (for example env=prod, role=primary)."

360 + },

361 + "queries": {

362 + "items": {

363 + "query": {

364 + "ui:widget": "textarea"

365 + }

366 + }

367 + },

368 + "metrics": {

369 + "items": {

370 + "ui:order": [

371 + "id",

372 + "query_ref",

373 + "query",

374 + "mode",

375 + "kv_mode",

376 + "labels_from_row",

377 + "charts"

378 + ],

379 + "mode": {

380 + "ui:widget": "radio",

381 + "ui:options": {

382 + "inline": true

383 + }

384 + },

385 + "query": {

386 + "ui:widget": "textarea"

387 + },

388 + "charts": {

389 + "items": {

390 + "type": {

391 + "ui:widget": "radio",

392 + "ui:options": {

393 + "inline": true

394 + }

395 + }

396 + }

397 + }

398 + }

399 + }

400 + }

401 +}

src/go/plugin/go.d/collector/sql/driver.go new

+17

@@ -0,0 +1,17 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +package sql

4 +

5 +import (

6 + _ "github.com/go-sql-driver/mysql"

7 + _ "github.com/jackc/pgx/v5/stdlib"

8 + _ "github.com/microsoft/go-mssqldb"

9 + _ "github.com/sijms/go-ora/v2"

10 +)

11 +

12 +var supportedDrivers = map[string]bool{

13 + "mysql": true,

14 + "oracle": true,

15 + "pgx": true,

16 + "sqlserver": true,

17 +}

src/go/plugin/go.d/collector/sql/metadata.yaml new

+529

@@ -0,0 +1,529 @@

1 +plugin_name: go.d.plugin

2 +modules:

3 + - meta:

4 + id: collector-go.d.plugin-sql

5 + plugin_name: go.d.plugin

6 + module_name: sql

7 + monitored_instance:

8 + name: SQL databases (generic)

9 + link: https://en.wikipedia.org/wiki/SQL

10 + categories:

11 + - data-collection.database-servers

12 + icon_filename: sql.svg

13 + related_resources:

14 + integrations:

15 + list: []

16 + alternative_monitored_instances: []

17 + info_provided_to_referring_integrations:

18 + description: ""

19 + keywords:

20 + - db

21 + - database

22 + - sql

23 + - mysql

24 + - maria

25 + - postgres

26 + - postgresql

27 + - pgx

28 + - oracle

29 + - sqlserver

30 + - mssql

31 + - generic

32 + most_popular: false

33 + overview:

34 + multi_instance: true

35 + data_collection:

36 + metrics_description: |

37 + Metrics and charts for this collector are **entirely defined by your SQL

38 + configuration**. There is no fixed metric reference: each job can expose

39 + different metrics depending on its `metrics` and `queries` blocks.

40 +

41 + To see what a specific job collects, open that job’s dashboard in Netdata

42 + and inspect the charts and dimensions it created.

43 +

44 + :::tip

45 +

46 + To change what is collected, edit the `metrics` (and optional `queries`)

47 + in the job configuration. After you save the changes, the updated set of

48 + charts and metrics is reflected in Netdata after the next data collection.

49 +

50 + :::

51 + method_description: |

52 + The collector connects to your database using Go’s **database/sql** package

53 + and the selected driver:

54 +

55 + - `mysql` — MySQL / MariaDB

56 + - `pgx` — PostgreSQL

57 + - `oracle` — Oracle Database

58 + - `sqlserver` — Microsoft SQL Server / Azure SQL

59 +

60 + For each metric block you define, it executes the SQL query (inline or via

61 + `query_ref`), reads the result set, and maps it to Netdata charts and

62 + dimensions.

63 +

64 + ### Result Processing Modes

65 +

66 + | Mode | How it works | Best used when |

67 + |------------|------------------------------------------------------------------------------|-----------------------------------------------------|

68 + | **columns**| Specific numeric columns from each row become dimensions on your charts. | The result set has stable, known column names. |

69 + | **kv** | One column provides metric names (keys) and another provides their values. | The set of metrics is dynamic or key–value shaped. |

70 + default_behavior:

71 + auto_detection:

72 + description: |

73 + This is a **generic collector** and does **not** perform automatic detection.

74 +

75 + It does not create any jobs on its own — you must configure at least one

76 + job before it can collect data.

77 + limits:

78 + description: |

79 + There are no built-in limits on the number of queries or rows processed.

80 + However, each metric block must define at least one chart, and each chart

81 + must define at least one dimension.

82 +

83 + Keep your queries lightweight and scoped to the data you actually need

84 + to avoid adding load on the database server.

85 + performance_impact:

86 + description: |

87 + Performance impact depends entirely on the queries you configure and the

88 + collection frequency (update_every).

89 +

90 + Prefer indexed reads, avoid full table scans or heavy aggregations, and

91 + consider using database views tailored for monitoring.

92 + additional_permissions:

93 + description: ""

94 + supported_platforms:

95 + include: []

96 + exclude: []

97 + setup:

98 + prerequisites:

99 + list:

100 + - title: Create a read-only database user

101 + description: |

102 + Create a dedicated user for Netdata with read-only privileges on the

103 + views/tables used in your monitoring queries.

104 +

105 + For example, on a typical RDBMS you would:

106 +

107 + - Create a user.

108 + - Grant SELECT on system metrics views or monitoring views.

109 +

110 + After creating the user and updating the configuration, restart the

111 + Netdata Agent with `sudo systemctl restart netdata`, or the appropriate

112 + method for your system.

113 + - title: Allow Netdata to connect to the database

114 + description: |

115 + Ensure the Netdata host can reach the database via the configured DSN,

116 + either using:

117 +

118 + - a local UNIX/TCP socket, or

119 + - a network connection (hostname/IP and port).

120 +

121 + If the database is remote, make sure any firewalls or security groups

122 + allow connections from the Netdata node.

123 + configuration:

124 + file:

125 + name: go.d/sql.conf

126 + options:

127 + description: |

128 + **Full Configuration Structure**

129 +

130 + ```yaml

131 + # ---------- CONNECTION ----------

132 + driver: <mysql|pgx|oracle|sqlserver> # REQUIRED. SQL driver.

133 + dsn: "<connection string>" # REQUIRED. Driver-specific DSN/URL.

134 +

135 + # Optional connection settings

136 + timeout: <seconds> # OPTIONAL. Query timeout.

137 +

138 + # Optional static labels applied to all charts

139 + static_labels:

140 + <label_key1>: <label_value>

141 + <label_key2>: <label_value>

142 +

143 + # ---------- REUSABLE QUERIES ----------

144 + # Optional. Define reusable SQL queries referenced later via query_ref.

145 + queries:

146 + - id: <query_id>

147 + query: |

148 + SELECT ...

149 +

150 + # ---------- METRICS ----------

151 + # Each metric block runs one query and generates one or more charts.

152 + metrics:

153 + - id: <metric_block_id> # REQUIRED. Unique within this job.

154 +

155 + # Choose ONE of these:

156 + query_ref: <query_id> # Use a reusable query

157 + # OR

158 + # query: | # Inline SQL

159 + # SELECT ...

160 +

161 + mode: <columns|kv> # REQUIRED. How to interpret result rows.

162 +

163 + # KV mode settings (only when mode: kv)

164 + kv_mode:

165 + name_col: <column_name> # Column containing keys

166 + value_col: <column_name> # Column containing numeric values

167 +

168 + # Optional: derive labels from row columns (creates per-label charts)

169 + labels_from_row:

170 + - source: <column_name> # Column name from result set

171 + name: <label_key> # Label key exposed to Netdata

172 + - source: <column_name>

173 + name: <label_key>

174 +

175 + # Charts produced by this metric block

176 + charts:

177 + - title: "<Chart Title>" # REQUIRED. Shown in dashboards.

178 + context: "<context.name>" # REQUIRED. Netdata context.

179 + family: "<family>" # REQUIRED. Netdata chart family.

180 + units: "<units>" # REQUIRED. Unit string for the chart.

181 + type: <line|stacked|area> # OPTIONAL. Default: line.

182 + algorithm: <absolute|incremental> # OPTIONAL. Default: absolute.

183 +

184 + dims:

185 + # ---- COLUMNS MODE DIM ----

186 + # In mode: columns, `source` MUST be a numeric COLUMN name from the result set.

187 + - name: <dim_id> # REQUIRED. Dimension id (unique within this chart).

188 + source: <column_name> # REQUIRED. Numeric column to chart.

189 +

190 + # ---- KV MODE DIM ----

191 + # In mode: kv, `source` MUST be a KEY name (NOT a column).

192 + # The collector finds the row where (row[kv_mode.name_col] == `source`)

193 + # and uses row[kv_mode.value_col].

194 + - name: <dim_id>

195 + source: <key_name> # REQUIRED. Key name resolved via kv_mode.name_col.

196 +

197 + # ---- STATUS DIM (one-hot 1/0) ----

198 + # Works in BOTH modes. Evaluates `status_when` against the resolved value:

199 + # * columns mode: the value in the specified column for the row

200 + # * kv mode: the value for the resolved key (row[kv_mode.value_col])

201 + - name: <dim_id>

202 + source: <column_name_or_key_name> # Same interpretation as above, per mode.

203 + status_when: # Exactly ONE of the following:

204 + equals: <string|number|bool> # Active (1) if value == this literal.

205 + # in: [ <v1>, <v2>, ... ] # Active if value is in the list.

206 + # match: '^regex$' # Active if value matches this regex.

207 + ```

208 + folding:

209 + title: Config options

210 + enabled: true

211 + list:

212 + - name: update_every

213 + description: Data collection interval (seconds).

214 + default_value: 1

215 + required: false

216 + group: Collection

217 + - name: autodetection_retry

218 + description: Autodetection retry interval (seconds). Not used for this collector. Set 0 to disable.

219 + default_value: 0

220 + required: false

221 + group: Collection

222 +

223 + - name: driver

224 + description: >

225 + SQL driver to use. Supported values: `mysql`, `pgx`, `oracle`, `sqlserver`.

226 + default_value: mysql

227 + required: true

228 + group: Target

229 + - name: dsn

230 + description: >

231 + Database connection string (DSN). The format depends on the selected driver (

232 + ([MySQL](https://github.com/go-sql-driver/mysql#dsn-data-source-name),

233 + [PostgreSQL](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING-URIS),

234 + [MS SQL Server](https://github.com/denisenkom/go-mssqldb#connection-parameters-and-dsn)).

235 + default_value: ""

236 + required: true

237 + group: Target

238 +

239 + - name: timeout

240 + description: Query and connection check timeout (seconds).

241 + default_value: 5

242 + required: false

243 + group: Connection

244 +

245 + - name: static_labels

246 + description: >

247 + A map of static labels added to every chart created by this job.

248 + Useful for tagging charts with environment, region, or role.

249 + default_value: "{}"

250 + required: false

251 + group: Labels

252 +

253 + - name: queries

254 + description: >

255 + A list of reusable queries. Metric blocks can reference these via `query_ref` to avoid repeating SQL. See [Configuration Structure](#configuration) for details.

256 + default_value: "[]"

257 + required: false

258 + group: Queries & Metrics

259 + - name: metrics

260 + description: >

261 + A list of metric blocks. Each block defines how a query is executed and how its result is transformed into one or more charts. See [Configuration Structure](#configuration) for details.

262 + default_value: "[]"

263 + required: true

264 + group: Queries & Metrics

265 +

266 + - name: vnode

267 + description: Associates this data collection job with a Virtual Node.

268 + default_value: ""

269 + required: false

270 + group: Virtual Node

271 + examples:

272 + folding:

273 + title: Config

274 + enabled: true

275 + list:

276 + - name: Columns mode – per-database conflicts (with labels)

277 + description: |

278 + PostgreSQL example that collects database-level conflict counters from

279 + `pg_stat_database_conflicts` and creates a separate chart instance per

280 + database using `labels_from_row`.

281 +

282 + The query:

283 +

284 + ```sql

285 + SELECT

286 + datname,

287 + confl_tablespace,

288 + confl_lock,

289 + confl_snapshot,

290 + confl_bufferpin,

291 + confl_deadlock

292 + FROM pg_stat_database_conflicts;

293 + ```

294 +

295 + Example output:

296 +

298 + |------------|------------------|------------|----------------|-----------------|----------------|

299 + | postgres | 0 | 0 | 0 | 0 | 0 |

300 + | production | 0 | 0 | 0 | 0 | 0 |

301 +

302 + This configuration turns each row into a **chart instance** (one for

303 + `db=postgres`, one for `db=production`) with five dimensions

304 + (`confl_tablespace`, `confl_lock`, `confl_snapshot`, `confl_bufferpin`,

305 + `confl_deadlock`).

306 + config: |

307 + jobs:

308 + - name: pg_conflicts_per_db

309 + driver: pgx

310 + dsn: 'postgresql://netdata:password@127.0.0.1:5432/postgres'

311 + timeout: 5

312 +

313 + metrics:

314 + - id: conflicts

315 + mode: columns

316 + query: |

317 + SELECT

318 + datname,

319 + confl_tablespace,

320 + confl_lock,

321 + confl_snapshot,

322 + confl_bufferpin,

323 + confl_deadlock

324 + FROM pg_stat_database_conflicts;

325 + labels_from_row:

326 + - source: datname

327 + name: db

328 + charts:

329 + - title: "PostgreSQL conflicts"

330 + context: sql.pg_conflicts

331 + family: conflicts

332 + units: conflicts

333 + type: line

334 + algorithm: absolute

335 + dims:

336 + - name: confl_tablespace

337 + source: confl_tablespace

338 + - name: confl_lock

339 + source: confl_lock

340 + - name: confl_snapshot

341 + source: confl_snapshot

342 + - name: confl_bufferpin

343 + source: confl_bufferpin

344 + - name: confl_deadlock

345 + source: confl_deadlock

346 +

347 + - name: Columns mode – single numeric value (uptime)

348 + description: |

349 + PostgreSQL example that exposes a single numeric metric (server uptime in

350 + seconds) as a one-dimension chart using columns mode.

351 +

352 + The query:

353 +

354 + ```sql

355 + SELECT

356 + EXTRACT(

357 + EPOCH FROM (now() - pg_postmaster_start_time())

358 + ) AS uptime_seconds;

359 + ```

360 +

361 + Example output:

362 +

363 + | uptime_seconds |

364 + |----------------|

365 + | 50.867359 |

366 +

367 + This configuration maps the `uptime_seconds` column to a single

368 + `uptime` dimension on the `sql.pg_uptime` chart.

369 + config: |

370 + jobs:

371 + - name: pg_uptime

372 + driver: pgx

373 + dsn: 'postgresql://netdata:password@127.0.0.1:5432/postgres'

374 + timeout: 5

375 +

376 + metrics:

377 + - id: uptime

378 + mode: columns

379 + query: |

380 + SELECT

381 + EXTRACT(

382 + EPOCH FROM (now() - pg_postmaster_start_time())

383 + ) AS uptime_seconds;

384 + charts:

385 + - title: "PostgreSQL uptime"

386 + context: sql.pg_uptime

387 + family: uptime

388 + units: seconds

389 + type: line

390 + algorithm: absolute

391 + dims:

392 + - name: uptime

393 + source: uptime_seconds

394 +

395 + - name: KV mode – connection states as key/value pairs

396 + description: |

397 + PostgreSQL example that aggregates connection states from

398 + `pg_stat_activity` and uses kv mode to map each state to a dimension.

399 +

400 + The query:

401 +

402 + ```sql

403 + SELECT

404 + state,

405 + count(*) AS cnt

406 + FROM pg_stat_activity

407 + GROUP BY state;

408 + ```

409 +

410 + Example output:

411 +

412 + | state | cnt |

413 + |------------------------------|-----|

414 + | active | 1 |

415 + | idle | 14 |

416 + | idle in transaction | 7 |

417 + | idle in transaction (aborted)| 1 |

418 + | fastpath function call | 1 |

419 + | disabled | 1 |

420 +

421 + With `mode: kv`, `state` becomes the **key** and `cnt` the **value**.

422 + Each distinct `state` value is mapped to a chart dimension via `dims[*].source`.

423 + config: |

424 + jobs:

425 + - name: pg_activity_states

426 + driver: pgx

427 + dsn: 'postgresql://netdata:password@127.0.0.1:5432/postgres'

428 + timeout: 5

429 +

430 + metrics:

431 + - id: activity_states

432 + mode: kv

433 + query: |

434 + SELECT

435 + state,

436 + count(*) AS cnt

437 + FROM pg_stat_activity

438 + GROUP BY state;

439 + kv_mode:

440 + name_col: state

441 + value_col: cnt

442 + charts:

443 + - title: "PostgreSQL connection states"

444 + context: sql.pg_activity_states

445 + family: connections

446 + units: connections

447 + type: stacked

448 + algorithm: absolute

449 + dims:

450 + - name: active

451 + source: active

452 + - name: idle

453 + source: idle

454 + - name: idle_in_transaction

455 + source: "idle in transaction"

456 + - name: idle_in_transaction_aborted

457 + source: "idle in transaction (aborted)"

458 + - name: fastpath_function_call

459 + source: "fastpath function call"

460 + - name: disabled

461 + source: disabled

462 +

463 + - name: Columns mode – map state values to a status metric

464 + description: |

465 + Simple PostgreSQL example that turns a boolean-like state into a 0/1

466 + status metric using `status_when`.

467 +

468 + The query:

469 +

470 + ```sql

471 + SELECT pg_is_in_recovery();

472 + ```

473 +

474 + Example output:

475 +

476 + | pg_is_in_recovery |

477 + |-------------------|

478 + | f |

479 +

480 + This configuration creates a single chart with two status dimensions:

481 + - `in_recovery` becomes **1 when the value is `"t"`** and **0 otherwise**.

482 + - `not_in_recovery` becomes **1 when the value is `"f"`** and **0 otherwise**.

483 + config: |

484 + jobs:

485 + - name: pg_recovery_status

486 + driver: pgx

487 + dsn: 'postgresql://netdata:password@127.0.0.1:5432/postgres'

488 + timeout: 5

489 +

490 + metrics:

491 + - id: recovery_status

492 + mode: columns

493 + query: |

494 + SELECT pg_is_in_recovery();

495 + charts:

496 + - title: "PostgreSQL recovery status"

497 + context: sql.pg_recovery_status

498 + family: state

499 + units: status

500 + type: line

501 + algorithm: absolute

502 + dims:

503 + - name: in_recovery

504 + source: pg_is_in_recovery

505 + status_when:

506 + equals: "t"

507 + - name: not_in_recovery

508 + source: pg_is_in_recovery

509 + status_when:

510 + equals: "f"

511 + troubleshooting:

512 + problems:

513 + list: []

514 + alerts: []

515 + metrics:

516 + folding:

517 + title: Metrics

518 + enabled: false

519 + description: |

520 + Metrics and charts are **defined by your SQL queries and metric blocks** at runtime. They differ by database engine, schema, and configuration, and may include, for example, connection counts, cache hit ratios, row throughput, lock statistics, or custom business KPIs. Use the **Metrics** tab on the job’s dashboard to see exactly what is collected for that job.

521 +

522 + :::tip

523 +

524 + To change what is collected, edit the `metrics` (and optionally `queries`) sections in `go.d/sql.conf` for the corresponding job. Each change is reflected in Netdata charts after the next data collection.

525 +

526 + :::

527 +

528 + availability: []

529 + scopes: []

src/go/plugin/go.d/collector/sql/query.go new

+117

@@ -0,0 +1,117 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +package sql

4 +

5 +import (

6 + "context"

7 + "fmt"

8 + "maps"

9 + "time"

10 +)

11 +

12 +// QueryRowsCache maps query_id -> slice of row maps (col -> string value)

13 +type QueryRowsCache map[string][]map[string]string

14 +

15 +// execReusableQueries runs all queries declared in Config.Queries (new schema)

16 +// and returns their full rowsets in memory. It also returns per-query durations (ms).

17 +func (c *Collector) execReusableQueries(ctx context.Context) (QueryRowsCache, map[string]int64, error) {

18 + cache := make(QueryRowsCache, len(c.Queries))

19 + durations := make(map[string]int64, len(c.Queries))

20 +

21 + for i, q := range c.Queries {

22 + if q.ID == "" {

23 + return nil, nil, fmt.Errorf("queries[%d] missing id", i+1)

24 + }

25 + if q.Query == "" {

26 + return nil, nil, fmt.Errorf("queries[%d] missing query", i+1)

27 + }

28 +

29 + rows, dur, err := c.runSQL(ctx, q.Query)

30 + if err != nil {

31 + return nil, nil, fmt.Errorf("query %q failed: %w", q.ID, err)

32 + }

33 + cache[q.ID] = rows

34 + durations[q.ID] = dur

35 + }

36 +

37 + return cache, durations, nil

38 +}

39 +

40 +// execMetricQueries resolves and executes the query for each metric block.

41 +// If a metric uses query_ref, it reuses rows from qcache (no re-query).

42 +// If a metric has inline query, it executes it and stores the rows.

43 +func (c *Collector) execMetricQueries(ctx context.Context, qcache QueryRowsCache) (QueryRowsCache, map[string]int64, error) {

44 + cache := make(QueryRowsCache, len(c.Metrics))

45 + durations := make(map[string]int64, len(c.Metrics))

46 +

47 + for i, m := range c.Metrics {

48 + if m.ID == "" {

49 + return nil, nil, fmt.Errorf("metrics[%d] missing id", i+1)

50 + }

51 +

52 + switch {

53 + case m.QueryRef != "":

54 + // reuse pre-fetched rows; duration is 0 because we didn't re-run

55 + rows, ok := qcache[m.QueryRef]

56 + if !ok {

57 + return nil, nil, fmt.Errorf("metrics[%d] query_ref %q not found in queries cache", i+1, m.QueryRef)

58 + }

59 + cache[m.ID] = rows

60 + durations[m.ID] = 0

61 + case m.Query != "":

62 + rows, dur, err := c.runSQL(ctx, m.Query)

63 + if err != nil {

64 + return nil, nil, fmt.Errorf("metrics[%d] (%q) query failed: %w", i+1, m.ID, err)

65 + }

66 + cache[m.ID] = rows

67 + durations[m.ID] = dur

68 + default:

69 + return nil, nil, fmt.Errorf("metrics[%d] must set one of query_ref or query", i+1)

70 + }

71 + }

72 +

73 + return cache, durations, nil

74 +}

75 +

76 +// runSQL executes a SQL statement with c.Timeout and returns the rowset as []map[col]value.

77 +// Duration is milliseconds from QueryContext() start to first successful return.

78 +func (c *Collector) runSQL(ctx context.Context, query string) ([]map[string]string, int64, error) {

79 + qctx := ctx

80 + cancel := func() {}

81 + if d := c.Timeout.Duration(); d > 0 {

82 + qctx, cancel = context.WithTimeout(ctx, d)

83 + }

84 + defer cancel()

85 +

86 + start := time.Now()

87 + rows, err := c.db.QueryContext(qctx, query)

88 + if err != nil {

89 + return nil, 0, err

90 + }

91 + defer func() { _ = rows.Close() }()

92 +

93 + duration := time.Since(start).Milliseconds()

94 +

95 + columns, err := rows.Columns()

96 + if err != nil {

97 + return nil, duration, err

98 + }

99 +

100 + scan := makeRawBytesSlice(len(columns))

101 + out := make([]map[string]string, 0, 64)

102 + row := make(map[string]string, len(columns))

103 +

104 + for rows.Next() {

105 + if err := rows.Scan(scan...); err != nil {

106 + return nil, duration, err

107 + }

108 + clear(row)

109 + for i := range columns {

110 + row[columns[i]] = rawBytesToString(scan[i])

111 + }

112 +

113 + out = append(out, maps.Clone(row))

114 + }

115 +

116 + return out, duration, rows.Err()

117 +}

src/go/plugin/go.d/collector/sql/testdata/config.json new

+58

@@ -0,0 +1,58 @@

1 +{

2 + "update_every": 123,

3 + "autodetection_retry": 123,

4 + "driver": "postgres",

5 + "dsn": "host=localhost port=5432 user=netdata dbname=metrics sslmode=disable",

6 + "timeout": 5,

7 + "static_labels": {

8 + "env": "prod",

9 + "region": "eu-west"

10 + },

11 + "queries": [

12 + {

13 + "id": "get_users",

14 + "query": "SELECT id, name FROM users"

15 + }

16 + ],

17 + "metrics": [

18 + {

19 + "id": "users_metrics",

20 + "query_ref": "get_users",

21 + "query": "query",

22 + "mode": "columns",

23 + "kv_mode": {

24 + "name_col": "metric_name",

25 + "value_col": "metric_value"

26 + },

27 + "labels_from_row": [

28 + {

29 + "source": "status",

30 + "name": "state"

31 + }

32 + ],

33 + "charts": [

34 + {

35 + "title": "User Count",

36 + "context": "app.users",

37 + "family": "users",

38 + "type": "line",

39 + "units": "count",

40 + "algorithm": "absolute",

41 + "dims": [

42 + {

43 + "name": "users_total",

44 + "source": "id",

45 + "status_when": {

46 + "equals": "equals",

47 + "in": [

48 + "in"

49 + ],

50 + "match": "match"

51 + }

52 + }

53 + ]

54 + }

55 + ]

56 + }

57 + ]

58 +}

src/go/plugin/go.d/collector/sql/testdata/config.yaml new

+40

@@ -0,0 +1,40 @@

1 +update_every: 123

2 +autodetection_retry: 123

3 +

4 +driver: "postgres"

5 +dsn: "host=localhost port=5432 user=netdata dbname=metrics sslmode=disable"

6 +timeout: 5

7 +

8 +static_labels:

9 + env: "prod"

10 + region: "eu-west"

11 +

12 +queries:

13 + - id: "get_users"

14 + query: "SELECT id, name FROM users"

15 +

16 +metrics:

17 + - id: "users_metrics"

18 + query_ref: "get_users"

19 + query: "query"

20 + mode: "columns"

21 + kv_mode:

22 + name_col: "metric_name"

23 + value_col: "metric_value"

24 + labels_from_row:

25 + - source: "status"

26 + name: "state"

27 + charts:

28 + - title: "User Count"

29 + context: "app.users"

30 + family: "users"

31 + type: "line"

32 + units: "count"

33 + algorithm: "absolute"

34 + dims:

35 + - name: "users_total"

36 + source: "id"

37 + status_when:

38 + equals: "equals"

39 + in: ["in"]

40 + match: "match"