@cryptotaxi247 / netdata-1 / commits / 8dd00ca3d

optimized json generation loop; some two stage json generation progress

Costa Tsaousis (ktsaou) committed Oct 25, 2015 at 14:58 UTC 8dd00ca3d2de1ac0df4ae34dff437b13fbb0c0e5
3 files changed +486 -509
src/rrd.c
+2 -2
@@ -17,7 +17,7 @@
17
18 #include "rrd.h"
19
20 -#define RRD_DEFAULT_GAP_INTERPOLATIONS 10
20 +#define RRD_DEFAULT_GAP_INTERPOLATIONS 1
21
22 // ----------------------------------------------------------------------------
23 // globals
@@ -421,7 +421,7 @@ RRDSET *rrdset_create(const char *type, const char *id, const char *name, const
421 st->last_collected_time.tv_usec = 0;
422 st->counter_done = 0;
423
424 - st->gap_when_lost_iterations_above = config_get_number(st->id, "gap when lost iterations above", RRD_DEFAULT_GAP_INTERPOLATIONS);
424 + st->gap_when_lost_iterations_above = config_get_number(st->id, "gap when lost iterations above", RRD_DEFAULT_GAP_INTERPOLATIONS) + 2;
425
426 st->dimensions_index.root = NULL;
427 st->dimensions_index.compar = rrddim_compare;
src/rrd.h
+6 -1
@@ -295,7 +295,12 @@ extern unsigned long long rrdset_done(RRDSET *st);
295 )))
296
297 // get the timestamp of a specific slot in the round robin database
298 -#define rrdset_slot2time(st, slot) ()
298 +#define rrdset_slot2time(st, slot) ( rrdset_last_entry_t(st) - \
299 + ((st)->update_every * ( \
300 + ( (slot) > rrdset_last_slot(st)) ? \
301 + ( (rrdset_last_slot(st) - slot + (st)->entries) ) : \
302 + ( (rrdset_last_slot(st) - slot) )) \
303 + ))
304
305 // ----------------------------------------------------------------------------
306 // RRD DIMENSION functions
src/rrd2json.c
+478 -506
@@ -160,474 +160,206 @@ void rrd_stats_all_json(struct web_buffer *wb)
160 );
161 }
162
163 -unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero)
164 -{
165 - int c;
166 - pthread_rwlock_rdlock(&st->rwlock);
163
164
169 - // -------------------------------------------------------------------------
170 - // switch from JSON to google JSON
171 -
172 - char kq[2] = "\"";
173 - char sq[2] = "\"";
174 - switch(type) {
175 - case DATASOURCE_GOOGLE_JSON:
176 - case DATASOURCE_GOOGLE_JSONP:
177 - kq[0] = '\0';
178 - sq[0] = '\'';
179 - break;
165 +// ----------------------------------------------------------------------------
166
181 - case DATASOURCE_JSON:
182 - default:
183 - break;
184 - }
167 +// RRDR options
168 +#define RRDR_EMPTY 0x01
169 +#define RRDR_RESET 0x02
170 +#define RRDR_HIDDEN 0x04
171 +#define RRDR_NONZERO 0x08
172
173
187 - // -------------------------------------------------------------------------
188 - // validate the parameters
189 -
190 - if(entries_to_show < 1) entries_to_show = 1;
191 - if(group < 1) group = 1;
192 -
193 - // find the oldest entry of the round-robin
194 - long max_entries_init = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
195 -
196 - time_t time_init = rrdset_last_entry_t(st);
197 -
198 - if(before == 0 || before > time_init) before = time_init;
199 - if(after == 0) after = rrdset_first_entry_t(st);
174 +typedef struct rrdresult {
175 + RRDSET *st; // the chart this result refers to
176
177 + int d; // the number of dimensions
178 + int n; // the number of values in the arrays
179
202 - // ---
180 + uint8_t *od; // the options for the dimensions
181
204 - // our return value (the last timestamp printed)
205 - // this is required to detect re-transmit in google JSONP
206 - time_t last_timestamp = 0;
182 + time_t *t; // array of n timestamps
183 + calculated_number *v; // array n x d values
184 + uint8_t *o; // array n x d options
185
186 + int c; // current line ( 0 ~ n )
187
209 - // -------------------------------------------------------------------------
210 - // find how many dimensions we have
211 -
212 - int dimensions = 0;
213 - RRDDIM *rd;
214 - for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
215 - if(!dimensions) {
216 - pthread_rwlock_unlock(&st->rwlock);
217 - web_buffer_printf(wb, "No dimensions yet.");
218 - return 0;
188 + int has_st_lock; // if st is read locked by us
189 +} RRDR;
190 +
191 +static void rrdr_dump(RRDR *r)
192 +{
193 + long c, i;
194 + RRDDIM *d;
195 +
196 + fprintf(stderr, "\nCHART %s (%s)\n", r->st->id, r->st->name);
197 +
198 + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
199 + fprintf(stderr, "DIMENSION %s (%s), %s%s%s%s\n"
200 + , d->id
201 + , d->name
202 + , (r->od[c] & RRDR_EMPTY)?"EMPTY ":""
203 + , (r->od[c] & RRDR_RESET)?"RESET ":""
204 + , (r->od[c] & RRDR_HIDDEN)?"HIDDEN ":""
205 + , (r->od[c] & RRDR_NONZERO)?"NONZERO ":""
206 + );
207 }
208
221 -
222 - // -------------------------------------------------------------------------
223 - // prepare various strings, to speed up the loop
224 -
225 - char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
226 - char normal_annotation[201]; snprintf(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
227 - char pre_date[51]; snprintf(pre_date, 50, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
228 - char post_date[21]; snprintf(post_date, 20, "%s}", sq);
229 - char pre_value[21]; snprintf(pre_value, 20, ",{%sv%s:", kq, kq);
230 - char post_value[21]; snprintf(post_value, 20, "}");
209 + if(r->c < 0) {
210 + fprintf(stderr, "RRDR does not have any values in it.\n");
211 + return;
212 + }
213
214 + fprintf(stderr, "RRDR includes %d values in it:\n", r->c + 1);
215 +
216 + // for each line in the array
217 + for(i = 0; i <= r->c ;i++) {
218 + calculated_number *cn = &r->v[ i * r->d ];
219 + uint8_t *co = &r->o[ i * r->d ];
220 +
221 + // print the id and the timestamp of the line
222 + fprintf(stderr, "%ld %ld ", i + 1, r->t[i]);
223 +
224 + // for each dimension
225 + for(c = 0, d = r->st->dimensions; d ;c++, d = d->next) {
226 + if(unlikely(r->od[c] & RRDR_HIDDEN)) continue;
227 + if(unlikely(!(r->od[c] & RRDR_NONZERO))) continue;
228 +
229 + if(co[c] & RRDR_EMPTY)
230 + fprintf(stderr, "null ");
231 + else
232 + fprintf(stderr, CALCULATED_NUMBER_FORMAT " %s%s%s%s "
233 + , cn[c]
234 + , (co[c] & RRDR_EMPTY)?"E":" "
235 + , (co[c] & RRDR_RESET)?"R":" "
236 + , (co[c] & RRDR_HIDDEN)?"H":" "
237 + , (co[c] & RRDR_NONZERO)?"N":" "
238 + );
239 + }
240
233 - // -------------------------------------------------------------------------
234 - // checks for debuging
235 -
236 - if(st->debug) {
237 - debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld"
238 - , st->id
239 - , rrdset_first_entry_t(st)
240 - , rrdset_last_entry_t(st)
241 - , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
242 - , after
243 - , before
244 - , before - after
245 - , entries_to_show
246 - , group
247 - , max_entries_init
248 - );
241 + fprintf(stderr, "\n");
242 + }
243 +}
244
250 - if(before < after)
251 - debug(D_RRD_STATS, "WARNING: %s The newest value in the database (%lu) is earlier than the oldest (%lu)", st->name, before, after);
245 +inline static calculated_number *rrdr_line_values(RRDR *r)
246 +{
247 + return &r->v[ r->c * r->d ];
248 +}
249
253 - if((before - after) > st->entries * st->update_every)
254 - debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
250 +inline static uint8_t *rrdr_line_options(RRDR *r)
251 +{
252 + return &r->o[ r->c * r->d ];
253 +}
254 +
255 +inline static int rrdr_line_init(RRDR *r, time_t t)
256 +{
257 + r->c++;
258 + if(unlikely(r->c >= r->n)) {
259 + r->c = r->n - 1;
260 + return 0;
261 }
262
263 + // save the time
264 + r->t[r->c] = t;
265
258 - // -------------------------------------------------------------------------
259 - // temp arrays for keeping values per dimension
260 -
261 - calculated_number group_values[dimensions]; // keep sums when grouping
262 - int print_hidden[dimensions]; // keep hidden flags
263 - int found_non_zero[dimensions];
264 - int found_non_existing[dimensions];
266 + return 1;
267 +}
268
266 - // initialize them
267 - for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
268 - group_values[c] = 0;
269 - print_hidden[c] = rd->hidden;
270 - found_non_zero[c] = 0;
271 - found_non_existing[c] = 0;
269 +inline static void rrdr_lock_rrdset(RRDR *r) {
270 + if(unlikely(!r)) {
271 + error("NULL value given!");
272 + return;
273 }
274
275 + pthread_rwlock_rdlock(&r->st->rwlock);
276 + r->has_st_lock = 1;
277 +}
278
275 - // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
276 - // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
279 +inline static void rrdr_unlock_rrdset(RRDR *r) {
280 + if(unlikely(!r)) {
281 + error("NULL value given!");
282 + return;
283 + }
284
278 - // -------------------------------------------------------------------------
279 - // remove dimensions that contain only zeros
285 + if(likely(r->has_st_lock)) {
286 + pthread_rwlock_unlock(&r->st->rwlock);
287 + r->has_st_lock = 0;
288 + }
289 +}
290
281 - int max_loop = 1;
282 - if(only_non_zero) max_loop = 2;
291 +inline static void rrdr_free(RRDR *r)
292 +{
293 + if(unlikely(!r)) {
294 + error("NULL value given!");
295 + return;
296 + }
297
284 - for(; max_loop ; max_loop--) {
298 + rrdr_unlock_rrdset(r);
299 + if(likely(r->t)) free(r->t);
300 + if(likely(r->v)) free(r->v);
301 + if(likely(r->o)) free(r->o);
302 + if(likely(r->od)) free(r->od);
303 + free(r);
304 +}
305
286 - // -------------------------------------------------------------------------
287 - // print the JSON header
288 -
289 - web_buffer_printf(wb, "{\n %scols%s:\n [\n", kq, kq);
290 - web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
291 - web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
292 - web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
306 +static RRDR *rrdr_create(RRDSET *st, int n)
307 +{
308 + if(unlikely(!st)) {
309 + error("NULL value given!");
310 + return NULL;
311 + }
312
294 - // print the header for each dimension
295 - // and update the print_hidden array for the dimensions that should be hidden
296 - int pc = 0;
297 - for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
298 - if(!print_hidden[c]) {
299 - pc++;
300 - web_buffer_printf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
301 - }
302 - }
303 - if(!pc) {
304 - web_buffer_printf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
305 - }
313 + RRDR *r = calloc(1, sizeof(RRDR));
314 + if(unlikely(!r)) goto cleanup;
315
307 - // print the begin of row data
308 - web_buffer_printf(wb, "\n ],\n %srows%s:\n [\n", kq, kq);
316 + r->st = st;
317
318 + rrdr_lock_rrdset(r);
319
311 - // -------------------------------------------------------------------------
312 - // the main loop
320 + RRDDIM *rd;
321 + for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
322
314 - int annotate_reset = 0;
315 - int annotation_count = 0;
316 -
317 - // to allow grouping on the same values, we need a pad
318 - long pad = before % group;
323 + r->n = n;
324 + r->t = malloc(n * sizeof(time_t));
325 + if(unlikely(!r->t)) goto cleanup;
326
320 - // the minimum line length we expect
321 - int line_size = 4096 + (dimensions * 200);
327 + r->t = malloc(n * sizeof(time_t));
328 + if(unlikely(!r->t)) goto cleanup;
329
323 - time_t now = time_init;
324 - long max_entries = max_entries_init;
330 + r->v = malloc(n * r->d * sizeof(calculated_number));
331 + if(unlikely(!r->v)) goto cleanup;
332
326 - long t;
333 + r->o = malloc(n * r->d * sizeof(uint8_t));
334 + if(unlikely(!r->o)) goto cleanup;
335
328 - long count = 0, printed = 0, group_count = 0;
329 - last_timestamp = 0;
336 + r->od = calloc(r->d, sizeof(uint8_t));
337 + if(unlikely(!r->od)) goto cleanup;
338
331 - long expected_to_start_at_slot = rrdset_time2slot(st, before);
339 + r->c = -1;
340
333 - for(t = rrdset_last_slot(st); max_entries ; now -= st->update_every, t--, max_entries--) {
334 - if(t < 0) t = st->entries - 1;
341 + return r;
342
336 - int print_this = 0;
343 +cleanup:
344 + error("Cannot allocate memory");
345 + if(likely(r)) rrdr_free(r);
346 + return NULL;
347 +}
348
338 - if(st->debug) {
339 - debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
340 - , st->id
341 - , t
342 - , count + 1
343 - , group_count + 1
344 - , printed
345 - , now
346 - , (((count + 1 - pad) % group) == 0)?"PRINT":" - "
347 - , (now >= after && now <= before)?"RANGE":" - "
348 - );
349 - }
349 +RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_method)
350 +{
351 + int debug = st->debug;
352
351 - // make sure we return data in the proper time range
352 - if(now < after || now > before) continue;
353 + time_t first_entry_t = rrdset_first_entry_t(st);
354 + time_t last_entry_t = rrdset_last_entry_t(st);
355
354 - if(expected_to_start_at_slot != -999999) {
355 - error("%s: Expected to start on slot %ld, started on %ld, %s", st->id, expected_to_start_at_slot, t, (t == expected_to_start_at_slot)?"OK":"ERROR");
356 - expected_to_start_at_slot = -999999;
357 - }
356 + // allow relative for before and after
357 + if(before <= st->update_every * st->entries) before = last_entry_t + before;
358 + if(after <= st->update_every * st->entries) after = last_entry_t + after;
359
359 - count++;
360 - group_count++;
361 -
362 - // check if we have to print this now
363 - if(((count - pad) % group) == 0) {
364 - if(printed >= entries_to_show) {
365 - // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
366 - break;
367 - }
368 -
369 - if(group_count != group) {
370 - // this is an incomplete group, skip it.
371 - for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
372 - group_values[c] = 0;
373 - found_non_existing[c] = 0;
374 - }
375 -
376 - group_count = 0;
377 - continue;
378 - }
379 -
380 - // check if we may exceed the buffer provided
381 - web_buffer_increase(wb, line_size);
382 -
383 - // generate the local date time
384 - struct tm *tm = localtime(&now);
385 - if(!tm) { error("localtime() failed."); continue; }
386 - if(now > last_timestamp) last_timestamp = now;
387 -
388 - if(printed) web_buffer_strcpy(wb, "]},\n");
389 - web_buffer_strcpy(wb, pre_date);
390 - web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
391 - web_buffer_strcpy(wb, post_date);
392 -
393 - print_this = 1;
394 - }
395 -
396 - // do the calculations
397 - for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
398 - storage_number n = rd->values[t];
399 - calculated_number value = unpack_storage_number(n);
400 -
401 - if(!does_storage_number_exist(n)) {
402 - value = 0.0;
403 - found_non_existing[c]++;
404 - }
405 - if(did_storage_number_reset(n)) annotate_reset = 1;
406 -
407 - switch(group_method) {
408 - case GROUP_MAX:
409 - if(abs(value) > abs(group_values[c])) group_values[c] = value;
410 - break;
411 -
412 - case GROUP_SUM:
413 - group_values[c] += value;
414 - break;
415 -
416 - default:
417 - case GROUP_AVERAGE:
418 - group_values[c] += value;
419 - if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
420 - break;
421 - }
422 - }
423 -
424 - if(print_this) {
425 - if(annotate_reset) {
426 - annotation_count++;
427 - web_buffer_strcpy(wb, overflow_annotation);
428 - annotate_reset = 0;
429 - }
430 - else
431 - web_buffer_strcpy(wb, normal_annotation);
432 -
433 - pc = 0;
434 - for(c = 0 ; c < dimensions ; c++) {
435 - if(found_non_existing[c] == group_count) {
436 - // all entries are non-existing
437 - pc++;
438 - web_buffer_strcpy(wb, pre_value);
439 - web_buffer_strcpy(wb, "null");
440 - web_buffer_strcpy(wb, post_value);
441 - }
442 - else if(!print_hidden[c]) {
443 - pc++;
444 - web_buffer_strcpy(wb, pre_value);
445 - web_buffer_rrd_value(wb, group_values[c]);
446 - web_buffer_strcpy(wb, post_value);
447 -
448 - if(group_values[c]) found_non_zero[c]++;
449 - }
450 -
451 - // reset them for the next loop
452 - group_values[c] = 0;
453 - found_non_existing[c] = 0;
454 - }
455 -
456 - // if all dimensions are hidden, print a null
457 - if(!pc) {
458 - web_buffer_strcpy(wb, pre_value);
459 - web_buffer_strcpy(wb, "null");
460 - web_buffer_strcpy(wb, post_value);
461 - }
462 -
463 - printed++;
464 - group_count = 0;
465 - }
466 - }
467 -
468 - if(printed) web_buffer_printf(wb, "]}");
469 - web_buffer_printf(wb, "\n ]\n}\n");
470 -
471 - if(only_non_zero && max_loop > 1) {
472 - int changed = 0;
473 - for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
474 - group_values[c] = 0;
475 - found_non_existing[c] = 0;
476 -
477 - if(!print_hidden[c] && !found_non_zero[c]) {
478 - changed = 1;
479 - print_hidden[c] = 1;
480 - }
481 - }
482 -
483 - if(changed) web_buffer_reset(wb);
484 - else break;
485 - }
486 - else break;
487 -
488 - } // max_loop
489 -
490 - debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->bytes);
491 -
492 - pthread_rwlock_unlock(&st->rwlock);
493 - return last_timestamp;
494 -}
495 -
496 -
497 -// ----------------------------------------------------------------------------
498 -
499 -// RRDR options
500 -#define RRDR_EMPTY 0x01
501 -#define RRDR_RESET 0x02
502 -#define RRDR_HIDDEN 0x04
503 -#define RRDR_NONZERO 0x08
504 -
505 -
506 -typedef struct rrdresult {
507 - RRDSET *st; // the chart this result refers to
508 -
509 - int d; // the number of dimensions
510 - int n; // the number of values in the arrays
511 -
512 - time_t *t; // array of timestamps
513 - calculated_number *v; // array n x d values
514 - uint8_t *o; // array n x d options
515 -
516 - int c; // current line (n)
517 -
518 - int has_st_lock; // if st is read locked by us
519 -} RRDR;
520 -
521 -inline static calculated_number *rrdr_line_values(RRDR *r)
522 -{
523 - return &r->v[ r->c * r->d ];
524 -}
525 -
526 -inline static uint8_t *rrdr_line_options(RRDR *r)
527 -{
528 - return &r->o[ r->c * r->d ];
529 -}
530 -
531 -inline static int rrdr_line_next(RRDR *r, time_t t)
532 -{
533 - // save the time
534 - r->t[r->c] = t;
535 -
536 - r->c++;
537 - if(unlikely(r->c >= r->n)) {
538 - r->c--;
539 - return 0;
540 - }
541 -
542 - return 1;
543 -}
544 -
545 -inline static void rrdr_lock_rrdset(RRDR *r) {
546 - if(unlikely(!r)) {
547 - error("NULL value given!");
548 - return;
549 - }
550 -
551 - pthread_rwlock_rdlock(&r->st->rwlock);
552 - r->has_st_lock = 1;
553 -}
554 -
555 -inline static void rrdr_unlock_rrdset(RRDR *r) {
556 - if(unlikely(!r)) {
557 - error("NULL value given!");
558 - return;
559 - }
560 -
561 - if(likely(r->has_st_lock)) {
562 - pthread_rwlock_unlock(&r->st->rwlock);
563 - r->has_st_lock = 0;
564 - }
565 -}
566 -
567 -inline static void rrdr_free(RRDR *r)
568 -{
569 - if(unlikely(!r)) {
570 - error("NULL value given!");
571 - return;
572 - }
573 -
574 - rrdr_unlock_rrdset(r);
575 - if(likely(r->t)) free(r->t);
576 - if(likely(r->v)) free(r->v);
577 - if(likely(r->o)) free(r->o);
578 - free(r);
579 -}
580 -
581 -static RRDR *rrdr_create(RRDSET *st, int n)
582 -{
583 - if(unlikely(!st)) {
584 - error("NULL value given!");
585 - return NULL;
586 - }
587 -
588 - RRDR *r = calloc(1, sizeof(RRDR));
589 - if(unlikely(!r)) goto cleanup;
590 -
591 - r->st = st;
592 -
593 - rrdr_lock_rrdset(r);
594 -
595 - RRDDIM *rd;
596 - for(rd = st->dimensions ; rd ; rd = rd->next) r->d++;
597 -
598 - r->n = n;
599 - r->t = malloc(sizeof(time_t) * n);
600 - if(unlikely(!r->t)) goto cleanup;
601 -
602 - r->t = malloc(sizeof(time_t) * n);
603 - if(unlikely(!r->t)) goto cleanup;
604 -
605 - r->v = malloc(sizeof(calculated_number) * n * r->d);
606 - if(unlikely(!r->v)) goto cleanup;
607 -
608 - r->o = malloc(sizeof(calculated_number) * n * r->d);
609 - if(unlikely(!r->o)) goto cleanup;
610 -
611 - return r;
612 -
613 -cleanup:
614 - error("Cannot allocate memory");
615 - if(likely(r)) rrdr_free(r);
616 - return NULL;
617 -}
618 -
619 -RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_method)
620 -{
621 - time_t first_entry_t = rrdset_first_entry_t(st);
622 - time_t last_entry_t = rrdset_last_entry_t(st);
623 -
624 - // allow relative for before and after
625 - if(before <= st->update_every * st->entries) before = last_entry_t + before;
626 - if(after <= st->update_every * st->entries) after = last_entry_t + after;
627 -
628 - // make sure they are within our timeframe
629 - if(before > last_entry_t) before = last_entry_t;
630 - if(before < first_entry_t) before = first_entry_t;
360 + // make sure they are within our timeframe
361 + if(before > last_entry_t) before = last_entry_t;
362 + if(before < first_entry_t) before = first_entry_t;
363
364 if(after > last_entry_t) after = last_entry_t;
365 if(after < first_entry_t) after = first_entry_t;
@@ -647,7 +379,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
379 if(points <= 0) points = duration;
380
381 // calculate proper grouping of source data
650 - int group = duration / points;
382 + long group = duration / points;
383 if(group <= 0) group = 1;
384 if(duration / group > points) group++;
385
@@ -681,17 +413,10 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
413 long dimensions = r->d;
414
415
684 - // -------------------------------------------------------------------------
685 -
686 - // how many entries can we use from the source data?
687 - long max_entries = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
688 -
689 -
416 // -------------------------------------------------------------------------
417 // checks for debugging
418
693 - if(st->debug) {
694 - debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld"
419 + if(debug) debug(D_RRD_STATS, "INFO %s first_t: %lu, last_t: %lu, all_duration: %lu, after: %lu, before: %lu, duration: %lu, points: %ld, group: %ld"
420 , st->id
421 , first_entry_t
422 , last_entry_t
@@ -701,9 +426,7 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
426 , duration
427 , points
428 , group
704 - , max_entries
429 );
706 - }
430
431
432 // -------------------------------------------------------------------------
@@ -729,35 +452,45 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
452 // -------------------------------------------------------------------------
453 // the main loop
454
732 - int debug = st->debug;
733 -
734 - time_t now = last_entry_t,
735 - dt = st->update_every;
736 -
455 long t = rrdset_time2slot(st, before), // rrdset_last_slot(st),
738 - count = 0,
456 + stop_at_t = rrdset_time2slot(st, after),
457 added = 0,
740 - group_count = group,
741 - add_this = 0;
458 + group_count = 0,
459 + add_this = 0,
460 + stop_now = 0;
461
743 - for( ; max_entries ; now -= dt, t--, max_entries--) {
744 - if(unlikely(t < 0)) c = st->entries - 1;
462 + time_t now = rrdset_slot2time(st, t),
463 + dt = st->update_every,
464 + group_start_t = 0;
465
746 - if(unlikely(debug)) debug(D_RRD_STATS, "%s c = %ld, count = %ld, group_count = %ld, added = %ld, now = %lu, %s %s"
466 + if(debug) debug(D_RRD_STATS, "BEGIN %s after_t: %lu (stop slot %ld), before_t: %lu (start slot %ld), start_t(now): %lu"
467 + , st->id
468 + , after
469 + , stop_at_t
470 + , before
471 + , t
472 + , now
473 + );
474 +
475 + for( ; !stop_now ; now -= dt, t--) {
476 + if(unlikely(t < 0)) c = st->entries - 1;
477 + if(t == stop_at_t) stop_now = 1;
478 +
479 + if(debug) debug(D_RRD_STATS, "ROW %s c: %ld, group_count: %ld, added: %ld, now: %lu, %s %s"
480 , st->id
481 , t
749 - , count + 1
482 , group_count + 1
483 , added
484 , now
753 - , (group_count == 0)?"PRINT":" - "
485 + , (group_count + 1 == group)?"PRINT":" - "
486 , (now >= after && now <= before)?"RANGE":" - "
487 );
488
489 // make sure we return data in the proper time range
758 - if(unlikely(now < after || now > before)) continue;
490 + if(unlikely(now > before)) continue;
491 + if(unlikely(now < after)) break;
492
760 - count++;
493 + if(group_count == 0) group_start_t = now;
494 group_count++;
495
496 if(unlikely(group_count == group)) {
@@ -792,22 +525,26 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
525 case GROUP_AVERAGE:
526 group_values[c] += value;
527 break;
795 -
796 - group_values[c] += value;
797 - break;
528 }
529 }
530
531 // added it
532 if(unlikely(add_this)) {
533 + if(!rrdr_line_init(r, group_start_t)) break;
534 +
535 calculated_number *cn = rrdr_line_values(r);
536 uint8_t *co = rrdr_line_options(r);
537
538 for(rd = st->dimensions, c = 0 ; likely(rd && c < dimensions) ; rd = rd->next, c++) {
807 - if(rd->hidden) group_options[c] |= RRDR_HIDDEN;
539
540 + // update the dimension options
541 + if(found_non_zero[c]) r->od[c] |= RRDR_NONZERO;
542 + if(rd->hidden) r->od[c] |= RRDR_HIDDEN;
543 +
544 + // store the specific point options
545 co[c] = group_options[c];
546
547 + // store the value
548 if(group_counts[c] == 0) {
549 cn[c] = 0.0;
550 co[c] |= RRDR_EMPTY;
@@ -829,24 +566,36 @@ RRDR *rrd2rrdr(RRDSET *st, long points, time_t after, time_t before, int group_m
566 group_count = 0;
567 add_this = 0;
568 }
832 -
833 - rrdr_line_next(r, now);
569 }
570
836 - return r;
571 + rrdr_dump(r);
572 + rrdr_free(r);
573 + return NULL;
574 }
575
576 +unsigned long rrd_stats_json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero)
577 +{
578 + int c;
579 + pthread_rwlock_rdlock(&st->rwlock);
580 +
581
840 -/*
841 -unsigned long rrdset2json(int type, RRDSET *st, struct web_buffer *wb, int entries_to_show, int group, int group_method, time_t after, time_t before, int only_non_zero) {
582 + // -------------------------------------------------------------------------
583 + // switch from JSON to google JSON
584
843 - if(!st->dimensions) {
844 - web_buffer_printf(wb, "No dimensions yet.");
845 - return 0;
585 + char kq[2] = "\"";
586 + char sq[2] = "\"";
587 + switch(type) {
588 + case DATASOURCE_GOOGLE_JSON:
589 + case DATASOURCE_GOOGLE_JSONP:
590 + kq[0] = '\0';
591 + sq[0] = '\'';
592 + break;
593 +
594 + case DATASOURCE_JSON:
595 + default:
596 + break;
597 }
598
848 - int c;
849 - pthread_rwlock_rdlock(&st->rwlock);
599
600 // -------------------------------------------------------------------------
601 // validate the parameters
@@ -854,35 +603,57 @@ unsigned long rrdset2json(int type, RRDSET *st, struct web_buffer *wb, int entri
603 if(entries_to_show < 1) entries_to_show = 1;
604 if(group < 1) group = 1;
605
857 - // make sure current_entry is within limits
858 - long current_entry = (long)st->current_entry - (long)1;
859 - if(current_entry < 0) current_entry = 0;
860 - else if(current_entry >= st->entries) current_entry = st->entries - 1;
606 + time_t time_init = rrdset_last_entry_t(st);
607 +
608 + if(before == 0 || before > time_init) before = time_init;
609 + if(after == 0) after = rrdset_first_entry_t(st);
610 +
611 +
612 + // ---
613 +
614 + // our return value (the last timestamp printed)
615 + // this is required to detect re-transmit in google JSONP
616 + time_t last_timestamp = 0;
617 +
618 +
619 + // -------------------------------------------------------------------------
620 + // find how many dimensions we have
621 +
622 + int dimensions = 0;
623 + RRDDIM *rd;
624 + for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
625 + if(!dimensions) {
626 + pthread_rwlock_unlock(&st->rwlock);
627 + web_buffer_printf(wb, "No dimensions yet.");
628 + return 0;
629 + }
630
862 - // find the oldest entry of the round-robin
863 - long max_entries = (st->counter < (unsigned long)st->entries) ? st->counter : (unsigned long)st->entries;
864 - if(entries_to_show > max_entries) entries_to_show = max_entries;
631
866 - if(before == 0 || before > st->last_updated.tv_sec) before = st->last_updated.tv_sec;
867 - if(after == 0 || after < rrdset_first_entry_t(st)) after = rrdset_first_entry_t(st);
632 + // -------------------------------------------------------------------------
633 + // prepare various strings, to speed up the loop
634 +
635 + char overflow_annotation[201]; snprintf(overflow_annotation, 200, ",{%sv%s:%sRESET OR OVERFLOW%s},{%sv%s:%sThe counters have been wrapped.%s}", kq, kq, sq, sq, kq, kq, sq, sq);
636 + char normal_annotation[201]; snprintf(normal_annotation, 200, ",{%sv%s:null},{%sv%s:null}", kq, kq, kq, kq);
637 + char pre_date[51]; snprintf(pre_date, 50, " {%sc%s:[{%sv%s:%s", kq, kq, kq, kq, sq);
638 + char post_date[21]; snprintf(post_date, 20, "%s}", sq);
639 + char pre_value[21]; snprintf(pre_value, 20, ",{%sv%s:", kq, kq);
640 + char post_value[21]; snprintf(post_value, 20, "}");
641
869 - if(entries_to_show > (before - after) / st->update_every / group) entries_to_show = (before - after) / st->update_every / group;
642
643 // -------------------------------------------------------------------------
644 // checks for debuging
645
646 if(st->debug) {
875 - debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu, max_entries = %ld"
647 + debug(D_RRD_STATS, "%s first_entry_t = %lu, last_entry_t = %lu, duration = %lu, after = %lu, before = %lu, duration = %lu, entries_to_show = %lu, group = %lu"
648 , st->id
649 , rrdset_first_entry_t(st)
878 - , st->last_updated.tv_sec
879 - , st->last_updated.tv_sec - rrdset_first_entry_t(st)
650 + , rrdset_last_entry_t(st)
651 + , rrdset_last_entry_t(st) - rrdset_first_entry_t(st)
652 , after
653 , before
654 , before - after
655 , entries_to_show
656 , group
885 - , max_entries
657 );
658
659 if(before < after)
@@ -892,40 +663,241 @@ unsigned long rrdset2json(int type, RRDSET *st, struct web_buffer *wb, int entri
663 debug(D_RRD_STATS, "WARNING: %s The time difference between the oldest and the newest entries (%lu) is higher than the capacity of the database (%lu)", st->name, before - after, st->entries * st->update_every);
664 }
665
666 +
667 // -------------------------------------------------------------------------
668 + // temp arrays for keeping values per dimension
669
897 - int dimensions = 0, i, j;
898 - RRDDIM *rd;
899 - for( rd = st->dimensions ; rd ; rd = rd->next) dimensions++;
670 + calculated_number group_values[dimensions]; // keep sums when grouping
671 + int print_hidden[dimensions]; // keep hidden flags
672 + int found_non_zero[dimensions];
673 + int found_non_existing[dimensions];
674
901 - uint32_t *annotations[dimensions];
902 - calculated_number *values[dimensions];
903 - int enabled[dimensions];
675 + // initialize them
676 + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
677 + group_values[c] = 0;
678 + print_hidden[c] = rd->hidden;
679 + found_non_zero[c] = 0;
680 + found_non_existing[c] = 0;
681 + }
682
905 - for( rd = st->dimensions, i = 0 ; rd && i < dimensions ; rd = rd->next, i++) {
906 - annotations[i] = malloc(sizeof(uint32_t) * entries_to_show);
907 - if(!annotations[i]) fatal("Cannot allocate %z bytes of memory.", sizeof(uint32_t) * entries_to_show);
683
909 - values[i] = malloc(sizeof(calculated_number) * entries_to_show);
910 - if(!values[i]) fatal("Cannot allocate %z bytes of memory.", sizeof(calculated_number) * entries_to_show);
684 + // error("OLD: points=%d after=%d before=%d group=%d, duration=%d", entries_to_show, before - (st->update_every * group * entries_to_show), before, group, before - after + 1);
685 + // rrd2array(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method, only_non_zero);
686 + // rrd2rrdr(st, entries_to_show, before - (st->update_every * group * entries_to_show), before, group_method);
687
912 - calculated_number sum = rrddim2values(st, rd, values, annotations, entries_to_show, group, group_method, after, before);
913 - if(only_non_zero && sum == 0) enabled[i] = 0;
914 - else enabled[i] = 1;
915 - }
688 + // -------------------------------------------------------------------------
689 + // remove dimensions that contain only zeros
690
917 - switch(type) {
918 - case DATASOURCE_GOOGLE_JSON:
919 - case DATASOURCE_GOOGLE_JSONP:
920 - //rrdvalues2googlejson(st, rd, values, annotations, enabled, entries_to_show, );
921 - break;
691 + int max_loop = 1;
692 + if(only_non_zero) max_loop = 2;
693
923 - case DATASOURCE_JSON:
924 - default:
925 - //rrdvalues2json();
926 - break;
927 - }
694 + for(; max_loop ; max_loop--) {
695
929 - //return ???;
696 + // -------------------------------------------------------------------------
697 + // print the JSON header
698 +
699 + web_buffer_printf(wb, "{\n %scols%s:\n [\n", kq, kq);
700 + web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%stime%s,%spattern%s:%s%s,%stype%s:%sdatetime%s},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq);
701 + web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotation%s}},\n", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
702 + web_buffer_printf(wb, " {%sid%s:%s%s,%slabel%s:%s%s,%spattern%s:%s%s,%stype%s:%sstring%s,%sp%s:{%srole%s:%sannotationText%s}}", kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, sq, sq, kq, kq, kq, kq, sq, sq);
703 +
704 + // print the header for each dimension
705 + // and update the print_hidden array for the dimensions that should be hidden
706 + int pc = 0;
707 + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
708 + if(!print_hidden[c]) {
709 + pc++;
710 + web_buffer_printf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, rd->name, sq, kq, kq, sq, sq, kq, kq, sq, sq);
711 + }
712 + }
713 + if(!pc) {
714 + web_buffer_printf(wb, ",\n {%sid%s:%s%s,%slabel%s:%s%s%s,%spattern%s:%s%s,%stype%s:%snumber%s}", kq, kq, sq, sq, kq, kq, sq, "no data", sq, kq, kq, sq, sq, kq, kq, sq, sq);
715 + }
716 +
717 + // print the begin of row data
718 + web_buffer_printf(wb, "\n ],\n %srows%s:\n [\n", kq, kq);
719 +
720 +
721 + // -------------------------------------------------------------------------
722 + // the main loop
723 +
724 + int annotate_reset = 0;
725 + int annotation_count = 0;
726 +
727 + // to allow grouping on the same values, we need a pad
728 + long pad = before % group;
729 +
730 + // the minimum line length we expect
731 + int line_size = 4096 + (dimensions * 200);
732 +
733 + time_t now = time_init;
734 +
735 + long t = rrdset_time2slot(st, before),
736 + stop_at_t = rrdset_time2slot(st, after),
737 + stop_now = 0;
738 +
739 + long count = 0, printed = 0, group_count = 0;
740 + last_timestamp = 0;
741 +
742 + for(; !stop_now ; now -= st->update_every, t--) {
743 + if(t < 0) t = st->entries - 1;
744 + if(t == stop_at_t) stop_now = 1;
745 +
746 + int print_this = 0;
747 +
748 + if(st->debug) {
749 + debug(D_RRD_STATS, "%s t = %ld, count = %ld, group_count = %ld, printed = %ld, now = %lu, %s %s"
750 + , st->id
751 + , t
752 + , count + 1
753 + , group_count + 1
754 + , printed
755 + , now
756 + , (((count + 1 - pad) % group) == 0)?"PRINT":" - "
757 + , (now >= after && now <= before)?"RANGE":" - "
758 + );
759 + }
760 +
761 + // make sure we return data in the proper time range
762 + if(now > before) continue;
763 + if(now < after) break;
764 +
765 + //if(rrdset_slot2time(st, t) != now)
766 + // error("%s: slot=%ld, now=%ld, slot2time=%ld, diff=%ld, last_entry_t=%ld, rrdset_last_slot=%ld", st->id, t, now, rrdset_slot2time(st,t), now - rrdset_slot2time(st,t), rrdset_last_entry_t(st), rrdset_last_slot(st));
767 +
768 + count++;
769 + group_count++;
770 +
771 + // check if we have to print this now
772 + if(((count - pad) % group) == 0) {
773 + if(printed >= entries_to_show) {
774 + // debug(D_RRD_STATS, "Already printed all rows. Stopping.");
775 + break;
776 + }
777 +
778 + if(group_count != group) {
779 + // this is an incomplete group, skip it.
780 + for( rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
781 + group_values[c] = 0;
782 + found_non_existing[c] = 0;
783 + }
784 +
785 + group_count = 0;
786 + continue;
787 + }
788 +
789 + // check if we may exceed the buffer provided
790 + web_buffer_increase(wb, line_size);
791 +
792 + // generate the local date time
793 + struct tm *tm = localtime(&now);
794 + if(!tm) { error("localtime() failed."); continue; }
795 + if(now > last_timestamp) last_timestamp = now;
796 +
797 + if(printed) web_buffer_strcpy(wb, "]},\n");
798 + web_buffer_strcpy(wb, pre_date);
799 + web_buffer_jsdate(wb, tm->tm_year + 1900, tm->tm_mon, tm->tm_mday, tm->tm_hour, tm->tm_min, tm->tm_sec);
800 + web_buffer_strcpy(wb, post_date);
801 +
802 + print_this = 1;
803 + }
804 +
805 + // do the calculations
806 + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
807 + storage_number n = rd->values[t];
808 + calculated_number value = unpack_storage_number(n);
809 +
810 + if(!does_storage_number_exist(n)) {
811 + value = 0.0;
812 + found_non_existing[c]++;
813 + }
814 + if(did_storage_number_reset(n)) annotate_reset = 1;
815 +
816 + switch(group_method) {
817 + case GROUP_MAX:
818 + if(abs(value) > abs(group_values[c])) group_values[c] = value;
819 + break;
820 +
821 + case GROUP_SUM:
822 + group_values[c] += value;
823 + break;
824 +
825 + default:
826 + case GROUP_AVERAGE:
827 + group_values[c] += value;
828 + if(print_this) group_values[c] /= ( group_count - found_non_existing[c] );
829 + break;
830 + }
831 + }
832 +
833 + if(print_this) {
834 + if(annotate_reset) {
835 + annotation_count++;
836 + web_buffer_strcpy(wb, overflow_annotation);
837 + annotate_reset = 0;
838 + }
839 + else
840 + web_buffer_strcpy(wb, normal_annotation);
841 +
842 + pc = 0;
843 + for(c = 0 ; c < dimensions ; c++) {
844 + if(found_non_existing[c] == group_count) {
845 + // all entries are non-existing
846 + pc++;
847 + web_buffer_strcpy(wb, pre_value);
848 + web_buffer_strcpy(wb, "null");
849 + web_buffer_strcpy(wb, post_value);
850 + }
851 + else if(!print_hidden[c]) {
852 + pc++;
853 + web_buffer_strcpy(wb, pre_value);
854 + web_buffer_rrd_value(wb, group_values[c]);
855 + web_buffer_strcpy(wb, post_value);
856 +
857 + if(group_values[c]) found_non_zero[c]++;
858 + }
859 +
860 + // reset them for the next loop
861 + group_values[c] = 0;
862 + found_non_existing[c] = 0;
863 + }
864 +
865 + // if all dimensions are hidden, print a null
866 + if(!pc) {
867 + web_buffer_strcpy(wb, pre_value);
868 + web_buffer_strcpy(wb, "null");
869 + web_buffer_strcpy(wb, post_value);
870 + }
871 +
872 + printed++;
873 + group_count = 0;
874 + }
875 + }
876 +
877 + if(printed) web_buffer_printf(wb, "]}");
878 + web_buffer_printf(wb, "\n ]\n}\n");
879 +
880 + if(only_non_zero && max_loop > 1) {
881 + int changed = 0;
882 + for(rd = st->dimensions, c = 0 ; rd && c < dimensions ; rd = rd->next, c++) {
883 + group_values[c] = 0;
884 + found_non_existing[c] = 0;
885 +
886 + if(!print_hidden[c] && !found_non_zero[c]) {
887 + changed = 1;
888 + print_hidden[c] = 1;
889 + }
890 + }
891 +
892 + if(changed) web_buffer_reset(wb);
893 + else break;
894 + }
895 + else break;
896 +
897 + } // max_loop
898 +
899 + debug(D_RRD_STATS, "RRD_STATS_JSON: %s total %ld bytes", st->name, wb->bytes);
900 +
901 + pthread_rwlock_unlock(&st->rwlock);
902 + return last_timestamp;
903 }
931 -*/