@cryptotaxi247 / netdata-1 / commits / ff232c524

send host variables to prometheus (#4200)

* send host variables to prometheus; fixes #4035 * make sure labels do not start with comma * embed the custom variables scopes into the structures; expose chart variables and the host level * check the type of variable when streaming host variables * added URL option variables=yes to enable sending host variables to prometheus; code cleanup to backends * removed _host_var from variables * added flags to tag alarm variables; set the last updated time of alarm variables

Costa Tsaousis committed Sep 17, 2018 at 04:09 UTC ff232c524a5015af629a0b66bdca5bb9949d2781

21 files changed +363 -228

conf.d/health.d/ipc.conf

+2 -2

@@ -5,7 +5,7 @@

5 on: system.ipc_semaphores

6 os: linux

7 hosts: *

8 - calc: $semaphores * 100 / $ipc.semaphores.max

8 + calc: $semaphores * 100 / $ipc_semaphores_max

9 units: %

10 every: 10s

11 warn: $this > (($status >= $WARNING) ? (70) : (80))

@@ -18,7 +18,7 @@

18 on: system.ipc_semaphore_arrays

19 os: linux

20 hosts: *

21 - calc: $arrays * 100 / $ipc.semaphores.arrays.max

21 + calc: $arrays * 100 / $ipc_semaphores_arrays_max

22 units: %

23 every: 10s

24 warn: $this > (($status >= $WARNING) ? (70) : (80))

conf.d/health.d/load.conf

+1 -1

@@ -8,7 +8,7 @@

8 on: system.load

9 os: linux

10 hosts: *

11 - calc: ($system.cpu.processors <= 2) ? ( 2 ) : ( $system.cpu.processors )

11 + calc: ($active_processors == nan or $active_processors == inf or $active_processors < 2) ? ( 2 ) : ( $active_processors )

12 units: cpus

13 every: 1m

14 info: trigger point for load average alarms

conf.d/health.d/netfilter.conf

+1 -1

@@ -19,7 +19,7 @@

19 os: linux

20 hosts: *

21 lookup: max -10s unaligned of connections

22 - calc: $this * 100 / $netfilter.conntrack.max

22 + calc: $this * 100 / $netfilter_conntrack_max

23 units: %

24 every: 10s

25 warn: $this > (($status >= $WARNING) ? (70) : (80))

src/adaptive_resortable_list.c

@@ -21,6 +21,15 @@ inline void arl_callback_str2kernel_uint_t(const char *name, uint32_t hash, cons

21 // fprintf(stderr, "name '%s' with hash %u and value '%s' is %llu\n", name, hash, value, (unsigned long long)*d);

22 }

24 +inline void arl_callback_ssize_t(const char *name, uint32_t hash, const char *value, void *dst) {

25 + (void)name;

26 + (void)hash;

27 +

28 + register ssize_t *d = dst;

29 + *d = (ssize_t)str2ll(value, NULL);

30 + // fprintf(stderr, "name '%s' with hash %u and value '%s' is %zd\n", name, hash, value, *d);

31 +}

32 +

33 // create a new ARL

34 ARL_BASE *arl_create(const char *name, void (*processor)(const char *, uint32_t, const char *, void *), size_t rechecks) {

35 ARL_BASE *base = callocz(1, sizeof(ARL_BASE));

src/adaptive_resortable_list.h

@@ -117,6 +117,7 @@ extern void arl_begin(ARL_BASE *base);

117

118 extern void arl_callback_str2ull(const char *name, uint32_t hash, const char *value, void *dst);

119 extern void arl_callback_str2kernel_uint_t(const char *name, uint32_t hash, const char *value, void *dst);

120 +extern void arl_callback_ssize_t(const char *name, uint32_t hash, const char *value, void *dst);

121

122 // check a keyword against the ARL

123 // this is to be called for each keyword read from source data

src/backend_prometheus.c

+128 -44

@@ -110,8 +110,77 @@ static inline char *prometheus_units_copy(char *d, const char *s, size_t usable)

110

111 #define PROMETHEUS_ELEMENT_MAX 256

112 #define PROMETHEUS_LABELS_MAX 1024

113 +#define PROMETHEUS_VARIABLE_MAX 256

114

114 -static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, uint32_t options, time_t after, time_t before, int allhosts, int help, int types, int names, int timestamps) {

115 +struct host_variables_callback_options {

116 + RRDHOST *host;

117 + BUFFER *wb;

118 + BACKEND_OPTIONS backend_options;

119 + PROMETHEUS_OUTPUT_OPTIONS output_options;

120 + const char *prefix;

121 + const char *labels;

122 + time_t now;

123 + int host_header_printed;

124 + char name[PROMETHEUS_VARIABLE_MAX+1];

125 +};

126 +

127 +static int print_host_variables(RRDVAR *rv, void *data) {

128 + struct host_variables_callback_options *opts = data;

129 +

130 + if(rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_CUSTOM_CHART_VAR)) {

131 + if(!opts->host_header_printed) {

132 + opts->host_header_printed = 1;

133 +

134 + if(opts->output_options & PROMETHEUS_OUTPUT_HELP) {

135 + buffer_sprintf(opts->wb, "\n# COMMENT global host and chart variables\n");

136 + }

137 + }

138 +

139 + calculated_number value = rrdvar2number(rv);

140 + if(isnan(value) || isinf(value)) {

141 + if(opts->output_options & PROMETHEUS_OUTPUT_HELP)

142 + buffer_sprintf(opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rv->name, (isnan(value))?"NAN":"INF");

143 +

144 + return 0;

145 + }

146 +

147 + char *label_pre = "";

148 + char *label_post = "";

149 + if(opts->labels && *opts->labels) {

150 + label_pre = "{";

151 + label_post = "}";

152 + }

153 +

154 + prometheus_name_copy(opts->name, rv->name, sizeof(opts->name));

155 +

156 + if(opts->output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)

157 + buffer_sprintf(opts->wb

158 + , "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n"

159 + , opts->prefix

160 + , opts->name

161 + , label_pre

162 + , opts->labels

163 + , label_post

164 + , value

165 + , ((rv->last_updated) ? rv->last_updated : opts->now) * 1000ULL

166 + );

167 + else

168 + buffer_sprintf(opts->wb, "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT "\n"

169 + , opts->prefix

170 + , opts->name

171 + , label_pre

172 + , opts->labels

173 + , label_post

174 + , value

175 + );

176 +

177 + return 1;

178 + }

179 +

180 + return 0;

181 +}

182 +

183 +static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, BACKEND_OPTIONS backend_options, time_t after, time_t before, int allhosts, PROMETHEUS_OUTPUT_OPTIONS output_options) {

184 rrdhost_rdlock(host);

185

186 char hostname[PROMETHEUS_ELEMENT_MAX + 1];

@@ -119,13 +188,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

188

189 char labels[PROMETHEUS_LABELS_MAX + 1] = "";

190 if(allhosts) {

122 - if(timestamps)

191 + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)

192 buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);

193 else

194 buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", hostname, host->program_name, host->program_version);

195

196 if(host->tags && *(host->tags)) {

128 - if(timestamps) {

197 + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {

198 buffer_sprintf(wb, "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS);

199

200 // deprecated, exists only for compatibility with older queries

@@ -143,13 +212,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

212 snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);

213 }

214 else {

146 - if(timestamps)

215 + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)

216 buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);

217 else

218 buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", hostname, host->program_name, host->program_version);

219

220 if(host->tags && *(host->tags)) {

152 - if(timestamps) {

221 + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {

222 buffer_sprintf(wb, "netdata_host_tags_info{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);

223

224 // deprecated, exists only for compatibility with older queries

@@ -164,6 +233,21 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

233 }

234 }

235

236 + // send custom variables set for the host

237 + if(output_options & PROMETHEUS_OUTPUT_VARIABLES){

238 + struct host_variables_callback_options opts = {

239 + .host = host,

240 + .wb = wb,

241 + .labels = (labels[0] == ',')?&labels[1]:labels,

242 + .backend_options = backend_options,

243 + .output_options = output_options,

244 + .prefix = prefix,

245 + .now = now_realtime_sec(),

246 + .host_header_printed = 0

247 + };

248 + foreach_host_variable_callback(host, print_host_variables, &opts);

249 + }

250 +

251 // for each chart

252 RRDSET *st;

253 rrdset_foreach_read(st, host) {

@@ -172,14 +256,14 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

256 char family[PROMETHEUS_ELEMENT_MAX + 1];

257 char units[PROMETHEUS_ELEMENT_MAX + 1] = "";

258

175 - prometheus_label_copy(chart, (names && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);

259 + prometheus_label_copy(chart, (output_options & PROMETHEUS_OUTPUT_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);

260 prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);

261 prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);

262

179 - if(likely(backends_can_send_rrdset(options, st))) {

263 + if(likely(backends_can_send_rrdset(backend_options, st))) {

264 rrdset_rdlock(st);

265

182 - int as_collected = ((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED);

266 + int as_collected = (BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED);

267 int homogeneus = 1;

268 if(as_collected) {

269 if(rrdset_flag_check(st, RRDSET_FLAG_HOMEGENEOUS_CHECK))

@@ -189,14 +273,14 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

273 homogeneus = 0;

274 }

275 else {

192 - if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE)

276 + if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE)

277 prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX);

278 }

279

196 - if(unlikely(help))

280 + if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP))

281 buffer_sprintf(wb, "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n"

282 , (homogeneus)?"homogeneus":"heterogeneous"

199 - , (names && st->name) ? st->name : st->id

283 + , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id

284 , st->context

285 , st->family

286 , st->units

@@ -224,18 +308,18 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

308 // all the dimensions of the chart, has the same algorithm, multiplier and divisor

309 // we add all dimensions as labels

310

227 - prometheus_label_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

311 + prometheus_label_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

312

229 - if(unlikely(help))

313 + if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP))

314 buffer_sprintf(wb

315 , "# COMMENT %s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"

316 , prefix

317 , context

318 , suffix

235 - , (names && st->name) ? st->name : st->id

319 + , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id

320 , st->context

321 , st->family

238 - , (names && rd->name) ? rd->name : rd->id

322 + , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id

323 , rd->multiplier

324 , rd->divisor

325 , h

@@ -243,7 +327,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

327 , t

328 );

329

246 - if(unlikely(types))

330 + if(unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))

331 buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s %s\n"

332 , prefix

333 , context

@@ -251,7 +335,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

335 , t

336 );

337

254 - if(timestamps)

338 + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)

339 buffer_sprintf(wb

340 , "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n"

341 , prefix

@@ -281,19 +365,19 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

365 // the dimensions of the chart, do not have the same algorithm, multiplier or divisor

366 // we create a metric per dimension

367

284 - prometheus_name_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

368 + prometheus_name_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

369

286 - if(unlikely(help))

370 + if(unlikely(output_options & PROMETHEUS_OUTPUT_HELP))

371 buffer_sprintf(wb

372 , "# COMMENT %s_%s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"

373 , prefix

374 , context

375 , dimension

376 , suffix

293 - , (names && st->name) ? st->name : st->id

377 + , (output_options & PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id

378 , st->context

379 , st->family

296 - , (names && rd->name) ? rd->name : rd->id

380 + , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id

381 , rd->multiplier

382 , rd->divisor

383 , h

@@ -301,7 +385,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

385 , t

386 );

387

304 - if(unlikely(types))

388 + if(unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))

389 buffer_sprintf(wb, "# COMMENT TYPE %s_%s_%s%s %s\n"

390 , prefix

391 , context

@@ -310,7 +394,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

394 , t

395 );

396

313 - if(timestamps)

397 + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)

398 buffer_sprintf(wb

399 , "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n"

400 , prefix

@@ -341,30 +425,30 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

425 // we need average or sum of the data

426

427 time_t first_t = after, last_t = before;

344 - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t);

428 + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);

429

430 if(!isnan(value) && !isinf(value)) {

431

348 - if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE)

432 + if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE)

433 suffix = "_average";

350 - else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM)

434 + else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM)

435 suffix = "_sum";

436

353 - prometheus_label_copy(dimension, (names && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

437 + prometheus_label_copy(dimension, (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

438

355 - if (unlikely(help))

439 + if (unlikely(output_options & PROMETHEUS_OUTPUT_HELP))

440 buffer_sprintf(wb, "# COMMENT %s_%s%s%s: dimension \"%s\", value is %s, gauge, dt %llu to %llu inclusive\n"

441 , prefix

442 , context

443 , units

444 , suffix

361 - , (names && rd->name) ? rd->name : rd->id

445 + , (output_options & PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id

446 , st->units

447 , (unsigned long long)first_t

448 , (unsigned long long)last_t

449 );

450

367 - if (unlikely(types))

451 + if (unlikely(output_options & PROMETHEUS_OUTPUT_TYPES))

452 buffer_sprintf(wb, "# COMMENT TYPE %s_%s%s%s gauge\n"

453 , prefix

454 , context

@@ -372,7 +456,7 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

456 , suffix

457 );

458

375 - if(timestamps)

459 + if(output_options & PROMETHEUS_OUTPUT_TIMESTAMPS)

460 buffer_sprintf(wb, "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT " %llu\n"

461 , prefix

462 , context

@@ -409,32 +493,32 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER

493 rrdhost_unlock(host);

494 }

495

412 -static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, uint32_t options, const char *server, time_t now, int help) {

496 +static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, BACKEND_OPTIONS backend_options, const char *server, time_t now, PROMETHEUS_OUTPUT_OPTIONS output_options) {

497 if(!server || !*server) server = "default";

498

499 time_t after = prometheus_server_last_access(server, host, now);

500

501 int first_seen = 0;

502 if(!after) {

419 - after = now - backend_update_every;

503 + after = now - global_backend_update_every;

504 first_seen = 1;

505 }

506

507 if(after > now) {

508 // oops! this should never happen

425 - after = now - backend_update_every;

509 + after = now - global_backend_update_every;

510 }

511

428 - if(help) {

512 + if(output_options & PROMETHEUS_OUTPUT_HELP) {

513 int show_range = 1;

514 char *mode;

431 - if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED) {

515 + if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED) {

516 mode = "as collected";

517 show_range = 0;

518 }

435 - else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AVERAGE)

519 + else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE)

520 mode = "average";

437 - else if((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM)

521 + else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM)

522 mode = "sum";

523 else

524 mode = "unknown";

@@ -457,24 +541,24 @@ static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, uint32_t

541 return after;

542 }

543

460 -void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names, int timestamps) {

544 +void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options) {

545 time_t before = now_realtime_sec();

546

547 // we start at the point we had stopped before

464 - time_t after = prometheus_preparation(host, wb, options, server, before, help);

548 + time_t after = prometheus_preparation(host, wb, backend_options, server, before, output_options);

549

466 - rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, options, after, before, 0, help, types, names, timestamps);

550 + rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, backend_options, after, before, 0, output_options);

551 }

552

469 -void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names, int timestamps) {

553 +void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options) {

554 time_t before = now_realtime_sec();

555

556 // we start at the point we had stopped before

473 - time_t after = prometheus_preparation(host, wb, options, server, before, help);

557 + time_t after = prometheus_preparation(host, wb, backend_options, server, before, output_options);

558

559 rrd_rdlock();

560 rrdhost_foreach_read(host) {

477 - rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, options, after, before, 1, help, types, names, timestamps);

561 + rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, backend_options, after, before, 1, output_options);

562 }

563 rrd_unlock();

564 }

src/backend_prometheus.h

+11 -5

@@ -1,12 +1,18 @@

1 // SPDX-License-Identifier: GPL-3.0+

2 -//

3 -// Created by costa on 09/07/17.

4 -//

3 #ifndef NETDATA_BACKEND_PROMETHEUS_H

4 #define NETDATA_BACKEND_PROMETHEUS_H

9 -extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names, int timestamps);

10 -extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, uint32_t options, int help, int types, int names, int timestamps);

6 +typedef enum prometheus_output_flags {

7 + PROMETHEUS_OUTPUT_NONE = 0,

8 + PROMETHEUS_OUTPUT_HELP = (1 << 0),

9 + PROMETHEUS_OUTPUT_TYPES = (1 << 1),

10 + PROMETHEUS_OUTPUT_NAMES = (1 << 2),

11 + PROMETHEUS_OUTPUT_TIMESTAMPS = (1 << 3),

12 + PROMETHEUS_OUTPUT_VARIABLES = (1 << 4)

13 +} PROMETHEUS_OUTPUT_OPTIONS;

14 +

15 +extern void rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options);

16 +extern void rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, PROMETHEUS_OUTPUT_OPTIONS output_options);

18 #endif //NETDATA_BACKEND_PROMETHEUS_H

src/backends.c

+79 -78

@@ -23,10 +23,9 @@

23 // 5. repeats the above forever.

24 //

26 -const char *backend_prefix = "netdata";

27 -int backend_send_names = 1;

28 -int backend_update_every = 10;

29 -uint32_t backend_options = BACKEND_SOURCE_DATA_AVERAGE;

26 +const char *global_backend_prefix = "netdata";

27 +int global_backend_update_every = 10;

28 +BACKEND_OPTIONS global_backend_options = BACKEND_SOURCE_DATA_AVERAGE | BACKEND_OPTION_SEND_NAMES;

30 // ----------------------------------------------------------------------------

31 // helper functions for backends

@@ -53,7 +52,7 @@ inline calculated_number backend_calculate_value_from_stored_data(

52 , RRDDIM *rd // the dimension

53 , time_t after // the start timestamp

54 , time_t before // the end timestamp

56 - , uint32_t options // BACKEND_SOURCE_* bitmap

55 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

56 , time_t *first_timestamp // the first point of the database used in this response

57 , time_t *last_timestamp // the timestamp that should be reported to backend

58 ) {

@@ -133,7 +132,7 @@ inline calculated_number backend_calculate_value_from_stored_data(

132 return NAN;

133 }

134

136 - if(unlikely((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_SUM))

135 + if(unlikely(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM))

136 return sum;

137

138 return sum / (calculated_number)counter;

@@ -173,27 +172,26 @@ static inline int format_dimension_collected_graphite_plaintext(

172 , RRDDIM *rd // the dimension

173 , time_t after // the start timestamp

174 , time_t before // the end timestamp

176 - , uint32_t options // BACKEND_SOURCE_* bitmap

175 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

176 ) {

177 (void)host;

178 (void)after;

179 (void)before;

181 - (void)options;

180

181 char chart_name[RRD_ID_LENGTH_MAX + 1];

182 char dimension_name[RRD_ID_LENGTH_MAX + 1];

185 - backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

186 - backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

183 + backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

184 + backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

185

186 buffer_sprintf(

187 b

190 - , "%s.%s.%s.%s " COLLECTED_NUMBER_FORMAT " %u\n"

188 + , "%s.%s.%s.%s " COLLECTED_NUMBER_FORMAT " %llu\n"

189 , prefix

190 , hostname

191 , chart_name

192 , dimension_name

193 , rd->last_collected_value

196 - , (uint32_t)rd->last_collected_time.tv_sec

194 + , (unsigned long long)rd->last_collected_time.tv_sec

195 );

196

197 return 1;

@@ -208,29 +206,29 @@ static inline int format_dimension_stored_graphite_plaintext(

206 , RRDDIM *rd // the dimension

207 , time_t after // the start timestamp

208 , time_t before // the end timestamp

211 - , uint32_t options // BACKEND_SOURCE_* bitmap

209 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

210 ) {

211 (void)host;

212

213 char chart_name[RRD_ID_LENGTH_MAX + 1];

214 char dimension_name[RRD_ID_LENGTH_MAX + 1];

217 - backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

218 - backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

215 + backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

216 + backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

217

218 time_t first_t = after, last_t = before;

221 - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t);

219 + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);

220

221 if(!isnan(value)) {

222

223 buffer_sprintf(

224 b

227 - , "%s.%s.%s.%s " CALCULATED_NUMBER_FORMAT " %u\n"

225 + , "%s.%s.%s.%s " CALCULATED_NUMBER_FORMAT " %llu\n"

226 , prefix

227 , hostname

228 , chart_name

229 , dimension_name

230 , value

233 - , (uint32_t) last_t

231 + , (unsigned long long) last_t

232 );

233

234 return 1;

@@ -255,25 +253,24 @@ static inline int format_dimension_collected_opentsdb_telnet(

253 , RRDDIM *rd // the dimension

254 , time_t after // the start timestamp

255 , time_t before // the end timestamp

258 - , uint32_t options // BACKEND_SOURCE_* bitmap

256 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

257 ) {

258 (void)host;

259 (void)after;

260 (void)before;

263 - (void)options;

261

262 char chart_name[RRD_ID_LENGTH_MAX + 1];

263 char dimension_name[RRD_ID_LENGTH_MAX + 1];

267 - backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

268 - backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

264 + backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

265 + backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

266

267 buffer_sprintf(

268 b

272 - , "put %s.%s.%s %u " COLLECTED_NUMBER_FORMAT " host=%s%s%s\n"

269 + , "put %s.%s.%s %llu " COLLECTED_NUMBER_FORMAT " host=%s%s%s\n"

270 , prefix

271 , chart_name

272 , dimension_name

276 - , (uint32_t)rd->last_collected_time.tv_sec

273 + , (unsigned long long)rd->last_collected_time.tv_sec

274 , rd->last_collected_value

275 , hostname

276 , (host->tags)?" ":""

@@ -292,27 +289,27 @@ static inline int format_dimension_stored_opentsdb_telnet(

289 , RRDDIM *rd // the dimension

290 , time_t after // the start timestamp

291 , time_t before // the end timestamp

295 - , uint32_t options // BACKEND_SOURCE_* bitmap

292 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

293 ) {

294 (void)host;

295

296 time_t first_t = after, last_t = before;

300 - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t);

297 + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);

298

299 char chart_name[RRD_ID_LENGTH_MAX + 1];

300 char dimension_name[RRD_ID_LENGTH_MAX + 1];

304 - backend_name_copy(chart_name, (backend_send_names && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

305 - backend_name_copy(dimension_name, (backend_send_names && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

301 + backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

302 + backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

303

304 if(!isnan(value)) {

305

306 buffer_sprintf(

307 b

311 - , "put %s.%s.%s %u " CALCULATED_NUMBER_FORMAT " host=%s%s%s\n"

308 + , "put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s\n"

309 , prefix

310 , chart_name

311 , dimension_name

315 - , (uint32_t) last_t

312 + , (unsigned long long) last_t

313 , value

314 , hostname

315 , (host->tags)?" ":""

@@ -341,12 +338,12 @@ static inline int format_dimension_collected_json_plaintext(

338 , RRDDIM *rd // the dimension

339 , time_t after // the start timestamp

340 , time_t before // the end timestamp

344 - , uint32_t options // BACKEND_SOURCE_* bitmap

341 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

342 ) {

343 (void)host;

344 (void)after;

345 (void)before;

349 - (void)options;

346 + (void)backend_options;

347

348 const char *tags_pre = "", *tags_post = "", *tags = host->tags;

349 if(!tags) tags = "";

@@ -378,7 +375,7 @@ static inline int format_dimension_collected_json_plaintext(

375 "\"name\":\"%s\","

376 "\"value\":" COLLECTED_NUMBER_FORMAT ","

377

381 - "\"timestamp\": %u}\n",

378 + "\"timestamp\": %llu}\n",

379 prefix,

380 hostname,

381 tags_pre, tags, tags_post,

@@ -394,7 +391,7 @@ static inline int format_dimension_collected_json_plaintext(

391 rd->name,

392 rd->last_collected_value,

393

397 - (uint32_t)rd->last_collected_time.tv_sec

394 + (unsigned long long) rd->last_collected_time.tv_sec

395 );

396

397 return 1;

@@ -409,12 +406,12 @@ static inline int format_dimension_stored_json_plaintext(

406 , RRDDIM *rd // the dimension

407 , time_t after // the start timestamp

408 , time_t before // the end timestamp

412 - , uint32_t options // BACKEND_SOURCE_* bitmap

409 + , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

410 ) {

411 (void)host;

412

413 time_t first_t = after, last_t = before;

417 - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, options, &first_t, &last_t);

414 + calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);

415

416 if(!isnan(value)) {

417 const char *tags_pre = "", *tags_post = "", *tags = host->tags;

@@ -447,7 +444,7 @@ static inline int format_dimension_stored_json_plaintext(

444 "\"name\":\"%s\","

445 "\"value\":" CALCULATED_NUMBER_FORMAT ","

446

450 - "\"timestamp\": %u}\n",

447 + "\"timestamp\": %llu}\n",

448 prefix,

449 hostname,

450 tags_pre, tags, tags_post,

@@ -461,9 +458,9 @@ static inline int format_dimension_stored_json_plaintext(

458

459 rd->id,

460 rd->name,

464 - value,

465 -

466 - (uint32_t) last_t

461 + value,

462 +

463 + (unsigned long long) last_t

464 );

465

466 return 1;

@@ -482,7 +479,7 @@ static inline int process_json_response(BUFFER *b) {

479 static SIMPLE_PATTERN *charts_pattern = NULL;

480 static SIMPLE_PATTERN *hosts_pattern = NULL;

481

485 -inline int backends_can_send_rrdset(uint32_t options, RRDSET *st) {

482 +inline int backends_can_send_rrdset(BACKEND_OPTIONS backend_options, RRDSET *st) {

483 RRDHOST *host = st->rrdhost;

484

485 if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE)))

@@ -504,7 +501,7 @@ inline int backends_can_send_rrdset(uint32_t options, RRDSET *st) {

501 return 0;

502 }

503

507 - if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !((options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED))) {

504 + if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED))) {

505 debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.", st->id, host->hostname, rrd_memory_mode_name(host->rrd_memory_mode));

506 return 0;

507 }

@@ -512,24 +509,24 @@ inline int backends_can_send_rrdset(uint32_t options, RRDSET *st) {

509 return 1;

510 }

511

515 -inline uint32_t backend_parse_data_source(const char *source, uint32_t mode) {

512 +inline BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options) {

513 if(!strcmp(source, "raw") || !strcmp(source, "as collected") || !strcmp(source, "as-collected") || !strcmp(source, "as_collected") || !strcmp(source, "ascollected")) {

517 - mode |= BACKEND_SOURCE_DATA_AS_COLLECTED;

518 - mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AS_COLLECTED);

514 + backend_options |= BACKEND_SOURCE_DATA_AS_COLLECTED;

515 + backend_options &= ~(BACKEND_OPTIONS_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AS_COLLECTED);

516 }

517 else if(!strcmp(source, "average")) {

521 - mode |= BACKEND_SOURCE_DATA_AVERAGE;

522 - mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AVERAGE);

518 + backend_options |= BACKEND_SOURCE_DATA_AVERAGE;

519 + backend_options &= ~(BACKEND_OPTIONS_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AVERAGE);

520 }

521 else if(!strcmp(source, "sum") || !strcmp(source, "volume")) {

525 - mode |= BACKEND_SOURCE_DATA_SUM;

526 - mode &= ~(BACKEND_SOURCE_BITS ^ BACKEND_SOURCE_DATA_SUM);

522 + backend_options |= BACKEND_SOURCE_DATA_SUM;

523 + backend_options &= ~(BACKEND_OPTIONS_SOURCE_BITS ^ BACKEND_SOURCE_DATA_SUM);

524 }

525 else {

526 error("BACKEND: invalid data source method '%s'.", source);

527 }

528

532 - return mode;

529 + return backend_options;

530 }

531

532 static void backends_main_cleanup(void *ptr) {

@@ -547,7 +544,7 @@ void *backends_main(void *ptr) {

544 int default_port = 0;

545 int sock = -1;

546 BUFFER *b = buffer_create(1), *response = buffer_create(1);

550 - int (*backend_request_formatter)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, uint32_t) = NULL;

547 + int (*backend_request_formatter)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS) = NULL;

548 int (*backend_response_checker)(BUFFER *) = NULL;

549

550 // ------------------------------------------------------------------------

@@ -557,35 +554,39 @@ void *backends_main(void *ptr) {

554 .tv_sec = 0,

555 .tv_usec = 0

556 };

560 - int enabled = config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", 0);

561 - const char *source = config_get(CONFIG_SECTION_BACKEND, "data source", "average");

562 - const char *type = config_get(CONFIG_SECTION_BACKEND, "type", "graphite");

563 - const char *destination = config_get(CONFIG_SECTION_BACKEND, "destination", "localhost");

564 - backend_prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata");

565 - const char *hostname = config_get(CONFIG_SECTION_BACKEND, "hostname", localhost->hostname);

566 - backend_update_every = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", backend_update_every);

567 - int buffer_on_failures = (int)config_get_number(CONFIG_SECTION_BACKEND, "buffer on failures", 10);

568 - long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", backend_update_every * 2 * 1000);

569 - backend_send_names = config_get_boolean(CONFIG_SECTION_BACKEND, "send names instead of ids", backend_send_names);

557 + int enabled = config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", 0);

558 + const char *source = config_get(CONFIG_SECTION_BACKEND, "data source", "average");

559 + const char *type = config_get(CONFIG_SECTION_BACKEND, "type", "graphite");

560 + const char *destination = config_get(CONFIG_SECTION_BACKEND, "destination", "localhost");

561 + global_backend_prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata");

562 + const char *hostname = config_get(CONFIG_SECTION_BACKEND, "hostname", localhost->hostname);

563 + global_backend_update_every = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", global_backend_update_every);

564 + int buffer_on_failures = (int)config_get_number(CONFIG_SECTION_BACKEND, "buffer on failures", 10);

565 + long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", global_backend_update_every * 2 * 1000);

566 +

567 + if(config_get_boolean(CONFIG_SECTION_BACKEND, "send names instead of ids", (global_backend_options & BACKEND_OPTION_SEND_NAMES)))

568 + global_backend_options |= BACKEND_OPTION_SEND_NAMES;

569 + else

570 + global_backend_options &= ~BACKEND_OPTION_SEND_NAMES;

571

572 charts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"), NULL, SIMPLE_PATTERN_EXACT);

572 - hosts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);

573 + hosts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);

574

575

576 // ------------------------------------------------------------------------

577 // validate configuration options

578 // and prepare for sending data to our backend

579

579 - backend_options = backend_parse_data_source(source, backend_options);

580 + global_backend_options = backend_parse_data_source(source, global_backend_options);

581

582 if(timeoutms < 1) {

582 - error("BACKEND: invalid timeout %ld ms given. Assuming %d ms.", timeoutms, backend_update_every * 2 * 1000);

583 - timeoutms = backend_update_every * 2 * 1000;

583 + error("BACKEND: invalid timeout %ld ms given. Assuming %d ms.", timeoutms, global_backend_update_every * 2 * 1000);

584 + timeoutms = global_backend_update_every * 2 * 1000;

585 }

586 timeout.tv_sec = (timeoutms * 1000) / 1000000;

587 timeout.tv_usec = (timeoutms * 1000) % 1000000;

588

588 - if(!enabled || backend_update_every < 1)

589 + if(!enabled || global_backend_update_every < 1)

590 goto cleanup;

591

592 // ------------------------------------------------------------------------

@@ -596,7 +597,7 @@ void *backends_main(void *ptr) {

597 default_port = 2003;

598 backend_response_checker = process_graphite_response;

599

599 - if((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)

600 + if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)

601 backend_request_formatter = format_dimension_collected_graphite_plaintext;

602 else

603 backend_request_formatter = format_dimension_stored_graphite_plaintext;

@@ -607,7 +608,7 @@ void *backends_main(void *ptr) {

608 default_port = 4242;

609 backend_response_checker = process_opentsdb_response;

610

610 - if((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)

611 + if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)

612 backend_request_formatter = format_dimension_collected_opentsdb_telnet;

613 else

614 backend_request_formatter = format_dimension_stored_opentsdb_telnet;

@@ -618,7 +619,7 @@ void *backends_main(void *ptr) {

619 default_port = 5448;

620 backend_response_checker = process_json_response;

621

621 - if ((backend_options & BACKEND_SOURCE_BITS) == BACKEND_SOURCE_DATA_AS_COLLECTED)

622 + if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)

623 backend_request_formatter = format_dimension_collected_json_plaintext;

624 else

625 backend_request_formatter = format_dimension_stored_json_plaintext;

@@ -655,18 +656,18 @@ void *backends_main(void *ptr) {

656 chart_backend_reconnects = 0,

657 chart_backend_latency = 0;

658

658 - RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", "backends", NULL, 130600, backend_update_every, RRDSET_TYPE_LINE);

659 + RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", "backends", NULL, 130600, global_backend_update_every, RRDSET_TYPE_LINE);

660 rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

661 rrddim_add(chart_metrics, "lost", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

662 rrddim_add(chart_metrics, "sent", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

663

663 - RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", "backends", NULL, 130610, backend_update_every, RRDSET_TYPE_AREA);

664 + RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KB", "backends", NULL, 130610, global_backend_update_every, RRDSET_TYPE_AREA);

665 rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

666 rrddim_add(chart_bytes, "lost", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

667 rrddim_add(chart_bytes, "sent", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

668 rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

669

669 - RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", "backends", NULL, 130630, backend_update_every, RRDSET_TYPE_LINE);

670 + RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", "backends", NULL, 130630, global_backend_update_every, RRDSET_TYPE_LINE);

671 rrddim_add(chart_ops, "write", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

672 rrddim_add(chart_ops, "discard", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

673 rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

@@ -680,11 +681,11 @@ void *backends_main(void *ptr) {

681 *

682 * issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html

683 *

683 - RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", "backends", NULL, 130620, backend_update_every, RRDSET_TYPE_AREA);

684 + RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", "backends", NULL, 130620, global_backend_update_every, RRDSET_TYPE_AREA);

685 rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);

686 */

687

687 - RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", "backends", NULL, 130630, backend_update_every, RRDSET_TYPE_STACKED);

688 + RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "NetData Backend Thread CPU usage", "milliseconds/s", "backends", NULL, 130630, global_backend_update_every, RRDSET_TYPE_STACKED);

689 rrddim_add(chart_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);

690 rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);

691

@@ -692,9 +693,9 @@ void *backends_main(void *ptr) {

693 // ------------------------------------------------------------------------

694 // prepare the backend main loop

695

695 - info("BACKEND: configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, backend_update_every, hostname, backend_prefix);

696 + info("BACKEND: configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, global_backend_update_every, hostname, global_backend_prefix);

697

697 - usec_t step_ut = backend_update_every * USEC_PER_SEC;

698 + usec_t step_ut = global_backend_update_every * USEC_PER_SEC;

699 time_t after = now_realtime_sec();

700 int failures = 0;

701 heartbeat_t hb;

@@ -747,7 +748,7 @@ void *backends_main(void *ptr) {

748

749 RRDSET *st;

750 rrdset_foreach_read(st, host) {

750 - if(likely(backends_can_send_rrdset(backend_options, st))) {

751 + if(likely(backends_can_send_rrdset(global_backend_options, st))) {

752 rrdset_rdlock(st);

753

754 count_charts++;

@@ -755,7 +756,7 @@ void *backends_main(void *ptr) {

756 RRDDIM *rd;

757 rrddim_foreach_read(rd, st) {

758 if (likely(rd->last_collected_time.tv_sec >= after)) {

758 - chart_buffered_metrics += backend_request_formatter(b, backend_prefix, host, __hostname, st, rd, after, before, backend_options);

759 + chart_buffered_metrics += backend_request_formatter(b, global_backend_prefix, host, __hostname, st, rd, after, before, global_backend_options);

760 count_dims++;

761 }

762 else {

src/backends.h

+16 -10

@@ -2,21 +2,27 @@

2 #ifndef NETDATA_BACKENDS_H

3 #define NETDATA_BACKENDS_H 1

5 -#define BACKEND_SOURCE_DATA_AS_COLLECTED 0x00000001

6 -#define BACKEND_SOURCE_DATA_AVERAGE 0x00000002

7 -#define BACKEND_SOURCE_DATA_SUM 0x00000004

5 +typedef enum backend_options {

6 + BACKEND_OPTION_NONE = 0,

9 -#define BACKEND_SOURCE_BITS (BACKEND_SOURCE_DATA_AS_COLLECTED|BACKEND_SOURCE_DATA_AVERAGE|BACKEND_SOURCE_DATA_SUM)

8 + BACKEND_SOURCE_DATA_AS_COLLECTED = (1 << 0),

9 + BACKEND_SOURCE_DATA_AVERAGE = (1 << 1),

10 + BACKEND_SOURCE_DATA_SUM = (1 << 2),

11 -extern int backend_send_names;

12 -extern int backend_update_every;

13 -extern uint32_t backend_options;

14 -extern const char *backend_prefix;

12 + BACKEND_OPTION_SEND_NAMES = (1 << 16)

13 +} BACKEND_OPTIONS;

14 +

15 +#define BACKEND_OPTIONS_SOURCE_BITS (BACKEND_SOURCE_DATA_AS_COLLECTED|BACKEND_SOURCE_DATA_AVERAGE|BACKEND_SOURCE_DATA_SUM)

16 +#define BACKEND_OPTIONS_DATA_SOURCE(backend_options) (backend_options & BACKEND_OPTIONS_SOURCE_BITS)

17 +

18 +extern int global_backend_update_every;

19 +extern BACKEND_OPTIONS global_backend_options;

20 +extern const char *global_backend_prefix;

22 extern void *backends_main(void *ptr);

18 -extern int backends_can_send_rrdset(uint32_t options, RRDSET *st);

19 -extern uint32_t backend_parse_data_source(const char *source, uint32_t mode);

24 +extern int backends_can_send_rrdset(BACKEND_OPTIONS options, RRDSET *st);

25 +extern BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS mode);

27 extern calculated_number backend_calculate_value_from_stored_data(

28 RRDSET *st // the chart

src/health.c

@@ -522,6 +522,11 @@ void *health_main(void *ptr) {

522 );

523

524 rc->value = rc->calculation->result;

525 +

526 + if(rc->local) rc->local->last_updated = now;

527 + if(rc->family) rc->family->last_updated = now;

528 + if(rc->hostid) rc->hostid->last_updated = now;

529 + if(rc->hostname) rc->hostname->last_updated = now;

530 }

531 }

532 }

src/health.h

+18 -10

@@ -11,11 +11,20 @@ typedef enum rrdvar_type {

11 RRDVAR_TYPE_TIME_T = 2,

12 RRDVAR_TYPE_COLLECTED = 3,

13 RRDVAR_TYPE_TOTAL = 4,

14 - RRDVAR_TYPE_INT = 5,

15 - RRDVAR_TYPE_CALCULATED_ALLOCATED = 6 // a custom variable, allocated on purpose (ie. not inherited from charts)

16 - // used only for custom host global variables

14 + RRDVAR_TYPE_INT = 5

15 } RRDVAR_TYPE;

17 +typedef enum rrdvar_options {

18 + RRDVAR_OPTION_DEFAULT = 0,

19 + RRDVAR_OPTION_ALLOCATED = (1 << 0), // the value ptr is allocated (not a reference)

20 + RRDVAR_OPTION_CUSTOM_HOST_VAR = (1 << 1), // this is a custom host variable, not associated with a dimension

21 + RRDVAR_OPTION_CUSTOM_CHART_VAR = (1 << 2), // this is a custom chart variable, not associated with a dimension

22 + RRDVAR_OPTION_RRDCALC_LOCAL_VAR = (1 << 3), // this is a an alarm variable, attached to a chart

23 + RRDVAR_OPTION_RRDCALC_FAMILY_VAR = (1 << 4), // this is a an alarm variable, attached to a family

24 + RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR = (1 << 5), // this is a an alarm variable, attached to the host, using the chart id

25 + RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR = (1 << 6), // this is a an alarm variable, attached to the host, using the chart name

26 +} RRDVAR_OPTIONS;

27 +

28 // the variables as stored in the variables indexes

29 // there are 3 indexes:

30 // 1. at each chart (RRDSET.rrdvar_root_index)

@@ -28,6 +37,8 @@ typedef struct rrdvar {

37 uint32_t hash;

39 RRDVAR_TYPE type;

40 + RRDVAR_OPTIONS options;

41 +

42 void *value;

44 time_t last_updated;

@@ -39,12 +50,6 @@ typedef struct rrdvar {

50 // This means, there will be no speed penalty for using

51 // these variables

42 -typedef enum rrdvar_options {

43 - RRDVAR_OPTION_DEFAULT = (0 << 0),

44 - RRDVAR_OPTION_ALLOCATED = (1 << 0) // the value ptr is allocated (not a reference)

45 - // future use

46 -} RRDVAR_OPTIONS;

47 -

53 typedef struct rrdsetvar {

54 char *variable; // variable name

55 uint32_t hash; // variable name hash

@@ -343,6 +348,8 @@ typedef struct alarm_log {

348

349 #include "rrd.h"

350

351 +extern calculated_number rrdvar2number(RRDVAR *rv);

352 +

353 extern void rrdsetvar_rename_all(RRDSET *st);

354 extern RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options);

355 extern void rrdsetvar_free(RRDSETVAR *rs);

@@ -369,6 +376,7 @@ void health_api_v1_chart_variables2json(RRDSET *st, BUFFER *buf);

376

377 extern RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name);

378 extern void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value);

379 +extern int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data);

380

381 extern RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name);

382 extern void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rv, calculated_number value);

@@ -429,7 +437,7 @@ extern int rrdvar_fix_name(char *variable);

437 extern RRDCALC *rrdcalc_create(RRDHOST *host, RRDCALCTEMPLATE *rt, const char *chart);

438 extern void rrdcalc_create_part2(RRDHOST *host, RRDCALC *rc);

439

432 -extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, void *value);

440 +extern RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value);

441 extern void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv);

442

443 extern void health_alarm_log_free_one_nochecks_nounlink(ALARM_ENTRY *ae);

src/ipc.c

+5 -5

@@ -167,7 +167,7 @@ int do_ipc(int update_every, usec_t dt) {

167 static int initialized = 0, read_limits_next = -1;

168 static struct ipc_limits limits;

169 static struct ipc_status status;

170 - static RRDSETVAR *arrays_max = NULL, *semaphores_max = NULL;

170 + static RRDVAR *arrays_max = NULL, *semaphores_max = NULL;

171 static RRDSET *st_semaphores = NULL, *st_arrays = NULL;

172 static RRDDIM *rd_semaphores = NULL, *rd_arrays = NULL;

173

@@ -224,8 +224,8 @@ int do_ipc(int update_every, usec_t dt) {

224 }

225

226 // variables

227 - semaphores_max = rrdsetvar_custom_chart_variable_create(st_semaphores, "ipc.semaphores.max");

228 - arrays_max = rrdsetvar_custom_chart_variable_create(st_arrays, "ipc.semaphores.arrays.max");

227 + semaphores_max = rrdvar_custom_host_variable_create(localhost, "ipc_semaphores_max");

228 + arrays_max = rrdvar_custom_host_variable_create(localhost, "ipc_semaphores_arrays_max");

229 }

230

231 if(unlikely(read_limits_next < 0)) {

@@ -233,8 +233,8 @@ int do_ipc(int update_every, usec_t dt) {

233 error("Unable to fetch semaphore limits.");

234 }

235 else {

236 - if(semaphores_max) rrdsetvar_custom_chart_variable_set(semaphores_max, limits.semmns);

237 - if(arrays_max) rrdsetvar_custom_chart_variable_set(arrays_max, limits.semmni);

236 + if(semaphores_max) rrdvar_custom_host_variable_set(localhost, semaphores_max, limits.semmns);

237 + if(arrays_max) rrdvar_custom_host_variable_set(localhost, arrays_max, limits.semmni);

238

239 st_arrays->red = limits.semmni;

240 st_semaphores->red = limits.semmns;

src/proc_net_snmp.c

+3 -6

@@ -103,7 +103,6 @@ int do_proc_net_snmp(int update_every, usec_t dt) {

103 *arl_udplite = NULL;

104

105 static RRDVAR *tcp_max_connections_var = NULL;

106 - static ssize_t last_max_connections = 0;

106

107 if(unlikely(!arl_ip)) {

108 do_ip_packets = config_get_boolean_ondemand("plugin:proc:/proc/net/snmp", "ipv4 packets", CONFIG_BOOLEAN_AUTO);

@@ -182,7 +181,7 @@ int do_proc_net_snmp(int update_every, usec_t dt) {

181 // arl_expect(arl_tcp, "RtoAlgorithm", &snmp_root.tcp_RtoAlgorithm);

182 // arl_expect(arl_tcp, "RtoMin", &snmp_root.tcp_RtoMin);

183 // arl_expect(arl_tcp, "RtoMax", &snmp_root.tcp_RtoMax);

185 - arl_expect(arl_tcp, "MaxConn", &snmp_root.tcp_MaxConn);

184 + arl_expect_custom(arl_tcp, "MaxConn", arl_callback_ssize_t, &snmp_root.tcp_MaxConn);

185 arl_expect(arl_tcp, "ActiveOpens", &snmp_root.tcp_ActiveOpens);

186 arl_expect(arl_tcp, "PassiveOpens", &snmp_root.tcp_PassiveOpens);

187 arl_expect(arl_tcp, "AttemptFails", &snmp_root.tcp_AttemptFails);

@@ -670,10 +669,8 @@ int do_proc_net_snmp(int update_every, usec_t dt) {

669

670 // --------------------------------------------------------------------

671

673 - if(snmp_root.tcp_MaxConn != last_max_connections) {

674 - last_max_connections = snmp_root.tcp_MaxConn;

675 - rrdvar_custom_host_variable_set(localhost, tcp_max_connections_var, last_max_connections);

676 - }

672 + // this is smart enough to update it, only when it is changed

673 + rrdvar_custom_host_variable_set(localhost, tcp_max_connections_var, snmp_root.tcp_MaxConn);

674

675 // --------------------------------------------------------------------

676

src/proc_stat.c

+3 -3

@@ -122,7 +122,7 @@ int do_proc_stat(int update_every, usec_t dt) {

122 static int do_cpu = -1, do_cpu_cores = -1, do_interrupts = -1, do_context = -1, do_forks = -1, do_processes = -1, do_core_throttle_count = -1, do_package_throttle_count = -1, do_scaling_cur_freq = -1;

123 static uint32_t hash_intr, hash_ctxt, hash_processes, hash_procs_running, hash_procs_blocked;

124 static char *core_throttle_count_filename = NULL, *package_throttle_count_filename = NULL, *scaling_cur_freq_filename = NULL;

125 - static RRDSETVAR *cpus_var = NULL;

125 + static RRDVAR *cpus_var = NULL;

126 size_t cores_found = (size_t)processors;

127

128 if(unlikely(do_cpu == -1)) {

@@ -314,7 +314,7 @@ int do_proc_stat(int update_every, usec_t dt) {

314 rrddim_hide(cpu_chart->st, "idle");

315

316 if(unlikely(core == 0 && cpus_var == NULL))

317 - cpus_var = rrdsetvar_custom_chart_variable_create(cpu_chart->st, "processors");

317 + cpus_var = rrdvar_custom_host_variable_create(localhost, "active_processors");

318 }

319 else rrdset_next(cpu_chart->st);

320

@@ -561,7 +561,7 @@ int do_proc_stat(int update_every, usec_t dt) {

561 }

562

563 if(cpus_var)

564 - rrdsetvar_custom_chart_variable_set(cpus_var, cores_found);

564 + rrdvar_custom_host_variable_set(localhost, cpus_var, cores_found);

565

566 return 0;

567 }

src/rrd.h

+2 -2

@@ -104,8 +104,8 @@ typedef struct rrdfamily RRDFAMILY;

104

105 typedef enum rrddim_flags {

106 RRDDIM_FLAG_NONE = 0,

107 - RRDDIM_FLAG_HIDDEN = 1 << 0, // this dimension will not be offered to callers

108 - RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = 1 << 1 // do not offer RESET or OVERFLOW info to callers

107 + RRDDIM_FLAG_HIDDEN = (1 << 0), // this dimension will not be offered to callers

108 + RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1) // do not offer RESET or OVERFLOW info to callers

109 } RRDDIM_FLAGS;

110

111 #ifdef HAVE_C___ATOMIC

src/rrdcalc.c

+7 -4

@@ -65,15 +65,18 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {

65 st->red = rc->red;

66 }

68 - rc->local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);

69 - rc->family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, &rc->value);

68 + rc->local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_LOCAL_VAR, &rc->value);

69 + rc->family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rc->name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_FAMILY_VAR, &rc->value);

71 char fullname[RRDVAR_MAX_LENGTH + 1];

72 snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->id, rc->name);

73 - rc->hostid = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);

73 + rc->hostid = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTID_VAR, &rc->value);

75 snprintfz(fullname, RRDVAR_MAX_LENGTH, "%s.%s", st->name, rc->name);

76 - rc->hostname = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, &rc->value);

76 + rc->hostname = rrdvar_create_and_index("host", &host->rrdvar_root_index, fullname, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR, &rc->value);

77 +

78 + if(rc->hostid && !rc->hostname)

79 + rc->hostid->options |= RRDVAR_OPTION_RRDCALC_HOST_CHARTNAME_VAR;

81 if(!rc->units) rc->units = strdupz(st->units);

src/rrddimvar.c

+10 -10

@@ -118,8 +118,8 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) {

118 // - $id

119 // - $name

120

121 - rs->var_local_id = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_id, rs->type, rs->value);

122 - rs->var_local_name = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_name, rs->type, rs->value);

121 + rs->var_local_id = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);

122 + rs->var_local_name = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);

123

124 // FAMILY VARIABLES FOR THIS DIMENSION

125 // -----------------------------------

@@ -130,10 +130,10 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) {

130 // - $chart-context.id

131 // - $chart-context.name

132

133 - rs->var_family_id = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_id, rs->type, rs->value);

134 - rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_name, rs->type, rs->value);

135 - rs->var_family_contextid = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextid, rs->type, rs->value);

136 - rs->var_family_contextname = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextname, rs->type, rs->value);

133 + rs->var_family_id = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_id, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);

134 + rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_name, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);

135 + rs->var_family_contextid = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);

136 + rs->var_family_contextname = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_contextname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);

137

138 // HOST VARIABLES FOR THIS DIMENSION

139 // -----------------------------------

@@ -144,10 +144,10 @@ static inline void rrddimvar_create_variables(RRDDIMVAR *rs) {

144 // - $chart-name.id

145 // - $chart-name.name

146

147 - rs->var_host_chartidid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidid, rs->type, rs->value);

148 - rs->var_host_chartidname = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidname, rs->type, rs->value);

149 - rs->var_host_chartnameid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnameid, rs->type, rs->value);

150 - rs->var_host_chartnamename = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnamename, rs->type, rs->value);

147 + rs->var_host_chartidid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);

148 + rs->var_host_chartidname = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullidname, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);

149 + rs->var_host_chartnameid = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnameid, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);

150 + rs->var_host_chartnamename = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullnamename, rs->type, RRDVAR_OPTION_DEFAULT, rs->value);

151 }

152

153 RRDDIMVAR *rrddimvar_create(RRDDIM *rd, RRDVAR_TYPE type, const char *prefix, const char *suffix, void *value, RRDVAR_OPTIONS options) {

src/rrdpush.c

+2 -2

@@ -135,7 +135,7 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {

135 // send the chart local custom variables

136 RRDSETVAR *rs;

137 for(rs = st->variables; rs ;rs = rs->next) {

138 - if(unlikely(rs->options && RRDVAR_OPTION_ALLOCATED)) {

138 + if(unlikely(rs->type == RRDVAR_TYPE_CALCULATED && rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR)) {

139 calculated_number *value = (calculated_number *) rs->value;

140

141 buffer_sprintf(

@@ -245,7 +245,7 @@ static int rrdpush_sender_thread_custom_host_variables_callback(void *rrdvar_ptr

245 RRDVAR *rv = (RRDVAR *)rrdvar_ptr;

246 RRDHOST *host = (RRDHOST *)host_ptr;

247

248 - if(unlikely(rv->type == RRDVAR_TYPE_CALCULATED_ALLOCATED)) {

248 + if(unlikely(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR && rv->type == RRDVAR_TYPE_CALCULATED)) {

249 rrdpush_sender_add_host_variable_to_buffer_nolock(host, rv);

250

251 // return 1, so that the traversal will return the number of variables sent

src/rrdsetvar.c

+13 -9

@@ -44,6 +44,10 @@ static inline void rrdsetvar_create_variables(RRDSETVAR *rs) {

44 RRDSET *st = rs->rrdset;

45 RRDHOST *host = st->rrdhost;

47 + RRDVAR_OPTIONS options = rs->options;

48 + if(rs->options & RRDVAR_OPTION_ALLOCATED)

49 + options &= ~ RRDVAR_OPTION_ALLOCATED;

50 +

51 // ------------------------------------------------------------------------

52 // free the old ones (if any)

@@ -61,17 +65,17 @@ static inline void rrdsetvar_create_variables(RRDSETVAR *rs) {

66 // ------------------------------------------------------------------------

67 // CHART

64 - rs->var_local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->variable, rs->type, rs->value);

68 + rs->var_local = rrdvar_create_and_index("local", &st->rrdvar_root_index, rs->variable, rs->type, options, rs->value);

70 // ------------------------------------------------------------------------

71 // FAMILY

68 - rs->var_family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullid, rs->type, rs->value);

69 - rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullname, rs->type, rs->value);

72 + rs->var_family = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value);

73 + rs->var_family_name = rrdvar_create_and_index("family", &st->rrdfamily->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value);

75 // ------------------------------------------------------------------------

76 // HOST

73 - rs->var_host = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullid, rs->type, rs->value);

74 - rs->var_host_name = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullname, rs->type, rs->value);

77 + rs->var_host = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullid, rs->type, options, rs->value);

78 + rs->var_host_name = rrdvar_create_and_index("host", &host->rrdvar_root_index, rs->key_fullname, rs->type, options, rs->value);

79 }

81 RRDSETVAR *rrdsetvar_create(RRDSET *st, const char *variable, RRDVAR_TYPE type, void *value, RRDVAR_OPTIONS options) {

@@ -144,7 +148,7 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)

148 for(rs = st->variables; rs ; rs = rs->next) {

149 if(hash == rs->hash && strcmp(n, rs->variable) == 0) {

150 rrdset_unlock(st);

147 - if(rs->options & RRDVAR_OPTION_ALLOCATED) {

151 + if(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) {

152 free(n);

153 return rs;

154 }

@@ -161,7 +165,7 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)

165 calculated_number *v = mallocz(sizeof(calculated_number));

166 *v = NAN;

167

164 - rs = rrdsetvar_create(st, n, RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED);

168 + rs = rrdsetvar_create(st, n, RRDVAR_TYPE_CALCULATED, v, RRDVAR_OPTION_ALLOCATED|RRDVAR_OPTION_CUSTOM_CHART_VAR);

169 rrdset_unlock(st);

170

171 free(n);

@@ -169,8 +173,8 @@ RRDSETVAR *rrdsetvar_custom_chart_variable_create(RRDSET *st, const char *name)

173 }

174

175 void rrdsetvar_custom_chart_variable_set(RRDSETVAR *rs, calculated_number value) {

172 - if(unlikely(!(rs->options & RRDVAR_OPTION_ALLOCATED))) {

173 - error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value);

176 + if(rs->type != RRDVAR_TYPE_CALCULATED || !(rs->options & RRDVAR_OPTION_CUSTOM_CHART_VAR) || !(rs->options & RRDVAR_OPTION_ALLOCATED)) {

177 + error("RRDSETVAR: requested to set variable '%s' of chart '%s' on host '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom chart one.", rs->variable, rs->rrdset->id, rs->rrdset->rrdhost->hostname, value);

178 }

179 else {

180 calculated_number *v = rs->value;

src/rrdvar.c

+14 -6

@@ -60,14 +60,14 @@ inline void rrdvar_free(RRDHOST *host, avl_tree_lock *tree, RRDVAR *rv) {

60 error("RRDVAR: Attempted to delete variable '%s' from host '%s', but it is not found.", rv->name, host->hostname);

61 }

63 - if(rv->type == RRDVAR_TYPE_CALCULATED_ALLOCATED)

63 + if(rv->options & RRDVAR_OPTION_ALLOCATED)

64 freez(rv->value);

66 freez(rv->name);

67 freez(rv);

68 }

70 -inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, void *value) {

70 +inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, const char *name, RRDVAR_TYPE type, RRDVAR_OPTIONS options, void *value) {

71 char *variable = strdupz(name);

72 rrdvar_fix_name(variable);

73 uint32_t hash = simple_hash(variable);

@@ -80,7 +80,9 @@ inline RRDVAR *rrdvar_create_and_index(const char *scope, avl_tree_lock *tree, c

80 rv->name = variable;

81 rv->hash = hash;

82 rv->type = type;

83 + rv->options = options;

84 rv->value = value;

85 + rv->last_updated = now_realtime_sec();

87 RRDVAR *ret = rrdvar_index_add(tree, rv);

88 if(unlikely(ret != rv)) {

@@ -132,7 +134,7 @@ static RRDVAR *rrdvar_custom_variable_create(const char *scope, avl_tree_lock *t

134 calculated_number *v = callocz(1, sizeof(calculated_number));

135 *v = NAN;

136

135 - RRDVAR *rv = rrdvar_create_and_index(scope, tree_lock, name, RRDVAR_TYPE_CALCULATED_ALLOCATED, v);

137 + RRDVAR *rv = rrdvar_create_and_index(scope, tree_lock, name, RRDVAR_TYPE_CALCULATED, RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_ALLOCATED, v);

138 if(unlikely(!rv)) {

139 free(v);

140 debug(D_VARIABLES, "Requested variable '%s' already exists - possibly 2 plugins are updating it at the same time.", name);

@@ -141,6 +143,7 @@ static RRDVAR *rrdvar_custom_variable_create(const char *scope, avl_tree_lock *t

143 rrdvar_fix_name(variable);

144 uint32_t hash = simple_hash(variable);

145

146 + // find the existing one to return it

147 rv = rrdvar_index_find(tree_lock, variable, hash);

148

149 freez(variable);

@@ -154,25 +157,30 @@ RRDVAR *rrdvar_custom_host_variable_create(RRDHOST *host, const char *name) {

157 }

158

159 void rrdvar_custom_host_variable_set(RRDHOST *host, RRDVAR *rv, calculated_number value) {

157 - if(rv->type != RRDVAR_TYPE_CALCULATED_ALLOCATED)

160 + if(rv->type != RRDVAR_TYPE_CALCULATED || !(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR) || !(rv->options & RRDVAR_OPTION_ALLOCATED))

161 error("requested to set variable '%s' to value " CALCULATED_NUMBER_FORMAT " but the variable is not a custom one.", rv->name, value);

162 else {

163 calculated_number *v = rv->value;

164 if(*v != value) {

165 *v = value;

166

167 + rv->last_updated = now_realtime_sec();

168 +

169 // if the host is streaming, send this variable upstream immediately

170 rrdpush_sender_send_this_host_variable_now(host, rv);

171 }

172 }

173 }

174

175 +int foreach_host_variable_callback(RRDHOST *host, int (*callback)(RRDVAR *rv, void *data), void *data) {

176 + return avl_traverse_lock(&host->rrdvar_root_index, (int (*)(void *, void *))callback, data);

177 +}

178 +

179 // ----------------------------------------------------------------------------

180 // RRDVAR lookup

181

173 -static calculated_number rrdvar2number(RRDVAR *rv) {

182 +calculated_number rrdvar2number(RRDVAR *rv) {

183 switch(rv->type) {

175 - case RRDVAR_TYPE_CALCULATED_ALLOCATED:

184 case RRDVAR_TYPE_CALCULATED: {

185 calculated_number *n = (calculated_number *)rv->value;

186 return *n;

src/web_api_v1.c

+33 -30

@@ -253,12 +253,26 @@ inline int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w,

253 return 200;

254 }

255

256 +struct prometheus_output_options {

257 + char *name;

258 + PROMETHEUS_OUTPUT_OPTIONS flag;

259 +} prometheus_output_flags_root[] = {

260 + { "help", PROMETHEUS_OUTPUT_HELP },

261 + { "types", PROMETHEUS_OUTPUT_TYPES },

262 + { "names", PROMETHEUS_OUTPUT_NAMES },

263 + { "timestamps", PROMETHEUS_OUTPUT_TIMESTAMPS },

264 + { "variables", PROMETHEUS_OUTPUT_VARIABLES },

265 +

266 + // terminator

267 + { NULL, PROMETHEUS_OUTPUT_NONE },

268 +};

269 +

270 inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url) {

271 int format = ALLMETRICS_SHELL;

258 - int help = 0, types = 0, timestamps = 1, names = backend_send_names; // prometheus options

272 const char *prometheus_server = w->client_ip;

260 - uint32_t prometheus_options = backend_options;

261 - const char *prometheus_prefix = backend_prefix;

273 + uint32_t prometheus_backend_options = global_backend_options;

274 + PROMETHEUS_OUTPUT_OPTIONS prometheus_output_options = PROMETHEUS_OUTPUT_TIMESTAMPS | ((global_backend_options & BACKEND_OPTION_SEND_NAMES)?PROMETHEUS_OUTPUT_NAMES:0);

275 + const char *prometheus_prefix = global_backend_prefix;

276

277 while(url) {

278 char *value = mystrsep(&url, "?&");

@@ -280,30 +294,6 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client

294 else

295 format = 0;

296 }

283 - else if(!strcmp(name, "help")) {

284 - if(!strcmp(value, "yes"))

285 - help = 1;

286 - else

287 - help = 0;

288 - }

289 - else if(!strcmp(name, "types")) {

290 - if(!strcmp(value, "yes"))

291 - types = 1;

292 - else

293 - types = 0;

294 - }

295 - else if(!strcmp(name, "names")) {

296 - if(!strcmp(value, "yes"))

297 - names = 1;

298 - else

299 - names = 0;

300 - }

301 - else if(!strcmp(name, "timestamps")) {

302 - if(!strcmp(value, "yes"))

303 - timestamps = 1;

304 - else

305 - timestamps = 0;

306 - }

297 else if(!strcmp(name, "server")) {

298 prometheus_server = value;

299 }

@@ -311,7 +301,20 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client

301 prometheus_prefix = value;

302 }

303 else if(!strcmp(name, "data") || !strcmp(name, "source") || !strcmp(name, "data source") || !strcmp(name, "data-source") || !strcmp(name, "data_source") || !strcmp(name, "datasource")) {

314 - prometheus_options = backend_parse_data_source(value, prometheus_options);

304 + prometheus_backend_options = backend_parse_data_source(value, prometheus_backend_options);

305 + }

306 + else {

307 + int i;

308 + for(i = 0; prometheus_output_flags_root[i].name ; i++) {

309 + if(!strcmp(name, prometheus_output_flags_root[i].name)) {

310 + if(!strcmp(value, "yes") || !strcmp(value, "1") || !strcmp(value, "true"))

311 + prometheus_output_options |= prometheus_output_flags_root[i].flag;

312 + else

313 + prometheus_output_options &= ~prometheus_output_flags_root[i].flag;

314 +

315 + break;

316 + }

317 + }

318 }

319 }

320

@@ -331,12 +334,12 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client

334

335 case ALLMETRICS_PROMETHEUS:

336 w->response.data->contenttype = CT_PROMETHEUS;

334 - rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(host, w->response.data, prometheus_server, prometheus_prefix, prometheus_options, help, types, names, timestamps);

337 + rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(host, w->response.data, prometheus_server, prometheus_prefix, prometheus_backend_options, prometheus_output_options);

338 return 200;

339

340 case ALLMETRICS_PROMETHEUS_ALL_HOSTS:

341 w->response.data->contenttype = CT_PROMETHEUS;

339 - rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(host, w->response.data, prometheus_server, prometheus_prefix, prometheus_options, help, types, names, timestamps);

342 + rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(host, w->response.data, prometheus_server, prometheus_prefix, prometheus_backend_options, prometheus_backend_options);

343 return 200;

344

345 default: