@cryptotaxi247 / netdata-1 / commits / cc6ff3568

POST Functions (#18611)

* fix functions payload parsing * remove raw facets field

Costa Tsaousis committed Sep 24, 2024 at 17:05 UTC cc6ff35685ad66ea6f8ed46446a16c1a6e606198

7 files changed +65 -20

src/collectors/systemd-journal.plugin/systemd-journal.c

+1 -1

@@ -1166,7 +1166,7 @@ void function_systemd_journal(const char *transaction, char *function, usec_t *s

1166 LOGS_QUERY_STATUS tmp_fqs = {

1167 .facets = lqs_facets_create(

1168 LQS_DEFAULT_ITEMS_PER_QUERY,

1169 - FACETS_OPTION_ALL_KEYS_FTS,

1169 + FACETS_OPTION_ALL_KEYS_FTS | FACETS_OPTION_HASH_IDS,

1170 SYSTEMD_ALWAYS_VISIBLE_KEYS,

1171 SYSTEMD_KEYS_INCLUDED_IN_FACETS,

1172 SYSTEMD_KEYS_EXCLUDED_FROM_FACETS,

src/collectors/windows-events.plugin/windows-events.c

+1 -1

@@ -1081,7 +1081,7 @@ void function_windows_events(const char *transaction, char *function, usec_t *st

1081 LOGS_QUERY_STATUS tmp_fqs = {

1082 .facets = lqs_facets_create(

1083 LQS_DEFAULT_ITEMS_PER_QUERY,

1084 - FACETS_OPTION_ALL_KEYS_FTS,

1084 + FACETS_OPTION_ALL_KEYS_FTS | FACETS_OPTION_HASH_IDS,

1085 WEVT_ALWAYS_VISIBLE_KEYS,

1086 WEVT_KEYS_INCLUDED_IN_FACETS,

1087 WEVT_KEYS_EXCLUDED_FROM_FACETS,

src/libnetdata/facets/facets.c

+35 -12

@@ -359,6 +359,28 @@ uint32_t facets_rows(FACETS *facets) {

359 return facets->items_to_return;

360 }

361

362 +static const char *facets_key_id(FACET_KEY *k) {

363 + if(k->facets->options & FACETS_OPTION_HASH_IDS)

364 + return hash_to_static_string(k->hash);

365 + else

366 + return k->name ? k->name : hash_to_static_string(k->hash);

367 +}

368 +

369 +static const char *facets_key_value_id(FACET_KEY *k, FACET_VALUE *v) {

370 + if(k->facets->options & FACETS_OPTION_HASH_IDS)

371 + return hash_to_static_string(v->hash);

372 + else

373 + return v->name ? v->name : hash_to_static_string(v->hash);

374 +}

375 +

376 +void facets_use_hashes_for_ids(FACETS *facets, bool set) {

377 + netdata_log_info("FACETS_OPTION_HASH_IDS = %s", set ? "true" : "false");

378 + if(set)

379 + facets->options |= FACETS_OPTION_HASH_IDS;

380 + else

381 + facets->options &= ~(FACETS_OPTION_HASH_IDS);

382 +}

383 +

384 // ----------------------------------------------------------------------------

385

386 static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row);

@@ -2346,7 +2368,7 @@ void facets_accepted_parameters_to_json_array(FACETS *facets, BUFFER *wb, bool w

2368 if (!k->values.enabled)

2369 continue;

2370

2349 - buffer_json_add_array_item_string(wb, hash_to_static_string(k->hash));

2371 + buffer_json_add_array_item_string(wb, facets_key_id(k));

2372 }

2373 foreach_key_in_facets_done(k);

2374 }

@@ -2526,8 +2548,8 @@ static uint32_t facets_sort_and_reorder_values(FACET_KEY *k) {

2548 return ret;

2549 }

2550

2529 -void facets_table_config(BUFFER *wb) {

2530 - buffer_json_member_add_boolean(wb, "show_ids", false); // do not show the column ids to the user

2551 +void facets_table_config(FACETS *facets, BUFFER *wb) {

2552 + buffer_json_member_add_boolean(wb, "show_ids", (facets->options & FACETS_OPTION_HASH_IDS) ? false : true);

2553 buffer_json_member_add_boolean(wb, "has_history", true); // enable date-time picker with after-before

2554

2555 buffer_json_member_add_object(wb, "pagination");

@@ -2543,8 +2565,9 @@ void facets_table_config(BUFFER *wb) {

2565 void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry) {

2566 facets->report.used_hashes_registry = used_hashes_registry;

2567

2568 + facets_table_config(facets, wb);

2569 +

2570 if(!(facets->options & FACETS_OPTION_DATA_ONLY)) {

2547 - facets_table_config(wb);

2571 facets_accepted_parameters_to_json_array(facets, wb, true);

2572 }

2573

@@ -2577,13 +2600,13 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)

2600 buffer_json_add_array_item_object(wb); // key

2601 {

2602 buffer_json_member_add_string(

2580 - wb, "id", hash_to_static_string(k->hash));

2603 + wb, "id", facets_key_id(k));

2604

2605 buffer_json_member_add_string(

2606 wb, "name",

2607 facets_key_name_cached(k, facets->report.used_hashes_registry));

2608

2586 - buffer_json_member_add_string(wb, "raw", k->name);

2609 + // buffer_json_member_add_string(wb, "raw", k->name);

2610

2611 if(!k->order) k->order = facets->order++;

2612 buffer_json_member_add_uint64(wb, "order", k->order);

@@ -2600,11 +2623,11 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)

2623

2624 buffer_json_add_array_item_object(wb);

2625 {

2603 - buffer_json_member_add_string(wb, "id", hash_to_static_string(v->hash));

2626 + buffer_json_member_add_string(wb, "id", facets_key_value_id(k, v));

2627

2628 facets_key_value_transformed(facets, k, v, tb, FACETS_TRANSFORM_FACET);

2629 buffer_json_member_add_string(wb, "name", buffer_tostring(tb));

2607 - buffer_json_member_add_string(wb, "raw", v->name);

2630 + // buffer_json_member_add_string(wb, "raw", v->name);

2631 buffer_json_member_add_uint64(wb, "count", v->final_facet_value_counter);

2632 buffer_json_member_add_uint64(wb, "order", v->order);

2633 }

@@ -2673,11 +2696,11 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)

2696 if (k->options & FACET_KEY_OPTION_PRETTY_XML)

2697 transform = RRDF_FIELD_TRANSFORM_XML;

2698

2676 - const char *hash_str = hash_to_static_string(k->hash);

2699 + const char *key_id = facets_key_id(k);

2700

2701 buffer_rrdf_table_add_field(

2702 wb, field_id++,

2680 - hash_str, k->name ? k->name : hash_str,

2703 + key_id, k->name ? k->name : key_id,

2704 RRDF_FIELD_TYPE_STRING,

2705 visual, transform, 0, NULL, NAN,

2706 RRDF_FIELD_SORT_FIXED,

@@ -2771,7 +2794,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)

2794 first_histogram_hash = k->hash;

2795

2796 buffer_json_add_array_item_object(wb);

2774 - buffer_json_member_add_string(wb, "id", hash_to_static_string(k->hash));

2797 + buffer_json_member_add_string(wb, "id", facets_key_id(k));

2798 buffer_json_member_add_string(wb, "name", k->name);

2799 buffer_json_member_add_uint64(wb, "order", k->order);

2800 buffer_json_object_close(wb);

@@ -2799,7 +2822,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)

2822 }

2823

2824 if(show_histogram) {

2802 - buffer_json_member_add_string(wb, "id", k ? hash_to_static_string(k->hash) : "");

2825 + buffer_json_member_add_string(wb, "id", k ? facets_key_id(k) : "");

2826 buffer_json_member_add_string(wb, "name", k ? k->name : "");

2827 buffer_json_member_add_object(wb, "chart");

2828 {

src/libnetdata/facets/facets.h

+4 -1

@@ -88,6 +88,7 @@ typedef enum __attribute__((packed)) {

88 FACETS_OPTION_DONT_SEND_EMPTY_VALUE_FACETS = (1 << 5), // empty facet values will not be included in the report

89 FACETS_OPTION_SORT_FACETS_ALPHABETICALLY = (1 << 6),

90 FACETS_OPTION_SHOW_DELTAS = (1 << 7),

91 + FACETS_OPTION_HASH_IDS = (1 << 8), // when set, the id of the facets, keys and values will be their hash

92 } FACETS_OPTIONS;

94 FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const char *visible_keys, const char *facet_keys, const char *non_facet_keys);

@@ -138,7 +139,7 @@ usec_t facets_row_oldest_ut(FACETS *facets);

139 usec_t facets_row_newest_ut(FACETS *facets);

140 uint32_t facets_rows(FACETS *facets);

141

141 -void facets_table_config(BUFFER *wb);

142 +void facets_table_config(FACETS *facets, BUFFER *wb);

143

144 const char *facets_severity_to_string(FACET_ROW_SEVERITY severity);

145

@@ -148,4 +149,6 @@ bool facets_foreach_selected_value_in_key(FACETS *facets, const char *key, size_

149 void facets_row_bin_data_set(FACETS *facets, void (*cleanup_cb)(void *data), void *data);

150 void *facets_row_bin_data_get(FACETS *facets __maybe_unused, FACET_ROW *row);

151

152 +void facets_use_hashes_for_ids(FACETS *facets, bool set);

153 +

154 #endif

src/libnetdata/facets/logs_query_status.h

+14 -3

@@ -354,6 +354,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char

354 if (json_object_object_get_ex(jobj, LQS_PARAMETER_SOURCE, &sources)) {

355 if (json_object_get_type(sources) != json_type_array) {

356 buffer_sprintf(error, "member '%s' is not an array", LQS_PARAMETER_SOURCE);

357 + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: '%s' is not an array", LQS_PARAMETER_SOURCE);

358 return false;

359 }

360

@@ -369,6 +370,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char

370

371 if (json_object_get_type(src) != json_type_string) {

372 buffer_sprintf(error, "sources array item %zu is not a string", i);

373 + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: sources array item %zu is not a string", i);

374 return false;

375 }

376

@@ -401,6 +403,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char

403 if (json_object_object_get_ex(jobj, LQS_PARAMETER_FACETS, &fcts)) {

404 if (json_object_get_type(sources) != json_type_array) {

405 buffer_sprintf(error, "member '%s' is not an array", LQS_PARAMETER_FACETS);

406 + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: '%s' is not an array", LQS_PARAMETER_FACETS);

407 return false;

408 }

409

@@ -415,6 +418,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char

418

419 if (json_object_get_type(fct) != json_type_string) {

420 buffer_sprintf(error, "facets array item %zu is not a string", i);

421 + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: facets array item %zu is not a string", i);

422 return false;

423 }

424

@@ -430,6 +434,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char

434 if (json_object_object_get_ex(jobj, "selections", &selections)) {

435 if (json_object_get_type(selections) != json_type_object) {

436 buffer_sprintf(error, "member 'selections' is not an object");

437 + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: '%s' is not an object", "selections");

438 return false;

439 }

440

@@ -438,6 +443,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char

443 json_object_object_foreach(selections, key, val) {

444 if (json_object_get_type(val) != json_type_array) {

445 buffer_sprintf(error, "selection '%s' is not an array", key);

446 + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: selection '%s' is not an array", key);

447 return false;

448 }

449

@@ -449,6 +455,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char

455

456 if (json_object_get_type(value_obj) != json_type_string) {

457 buffer_sprintf(error, "selection '%s' array item %zu is not a string", key, i);

458 + // nd_log(NDLS_COLLECTORS, NDLP_ERR, "POST payload: selection '%s' array item %zu is not a string", key, i);

459 return false;

460 }

461

@@ -468,6 +475,7 @@ static inline bool lqs_request_parse_json_payload(json_object *jobj, const char

475 buffer_json_object_close(wb); // selections

476 }

477

478 + facets_use_hashes_for_ids(facets, false);

479 rq->fields_are_ids = false;

480 return true;

481 }

@@ -476,6 +484,8 @@ static inline bool lqs_request_parse_POST(LOGS_QUERY_STATUS *lqs, BUFFER *wb, BU

484 FACETS *facets = lqs->facets;

485 LOGS_QUERY_REQUEST *rq = &lqs->rq;

486

487 + buffer_json_member_add_object(wb, "_request");

488 +

489 struct logs_query_data qd = {

490 .transaction = transaction,

491 .facets = facets,

@@ -498,7 +508,7 @@ static inline bool lqs_request_parse_GET(LOGS_QUERY_STATUS *lqs, BUFFER *wb, cha

508 buffer_json_member_add_object(wb, "_request");

509

510 char func_copy[strlen(function) + 1];

501 - strcpy(func_copy, function);

511 + memcpy(func_copy, function, sizeof(func_copy));

512

513 char *words[LQS_MAX_PARAMS] = { NULL };

514 size_t num_words = quoted_strings_splitter_pluginsd(func_copy, words, LQS_MAX_PARAMS);

@@ -661,6 +671,7 @@ static inline bool lqs_request_parse_GET(LOGS_QUERY_STATUS *lqs, BUFFER *wb, cha

671 }

672 }

673

674 + facets_use_hashes_for_ids(facets, true);

675 rq->fields_are_ids = true;

676 return true;

677 }

@@ -689,7 +700,7 @@ static inline void lqs_info_response(BUFFER *wb, FACETS *facets) {

700 }

701 buffer_json_array_close(wb); // required_params array

702

692 - facets_table_config(wb);

703 + facets_table_config(facets, wb);

704

705 buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);

706 buffer_json_member_add_string(wb, "type", "table");

@@ -702,7 +713,7 @@ static inline void lqs_info_response(BUFFER *wb, FACETS *facets) {

713

714 static inline BUFFER *lqs_create_output_buffer(void) {

715 BUFFER *wb = buffer_create(0, NULL);

705 - buffer_flush(wb);

716 + buffer_reset(wb);

717 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);

718 return wb;

719 }

src/libnetdata/functions_evloop/functions_evloop.c

+9 -1

@@ -137,6 +137,8 @@ static void worker_add_job(struct functions_evloop_globals *wg, const char *keyw

137 function?function:"(unset)");

138 }

139 else {

140 + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "WORKER JOB WITH PAYLOAD '%s'", payload ? buffer_tostring(payload) : "NONE");

141 +

142 int timeout = str2i(timeout_s);

143

144 const char *msg = "No function with this name found";

@@ -222,6 +224,8 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {

224 char *s = (char *)buffer_tostring(buffer);

225

226 if(strstr(&s[deferred.last_len], PLUGINSD_CALL_FUNCTION_PAYLOAD_END "\n") != NULL) {

227 + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION PAYLOAD END");

228 +

229 if(deferred.last_len > 0)

230 // remove the trailing newline from the buffer

231 deferred.last_len--;

@@ -254,6 +258,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {

258 const char *keyword = get_word(words, num_words, 0);

259

260 if(keyword && (strcmp(keyword, PLUGINSD_CALL_FUNCTION) == 0)) {

261 + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION CALL");

262 char *transaction = get_word(words, num_words, 1);

263 char *timeout_s = get_word(words, num_words, 2);

264 char *function = get_word(words, num_words, 3);

@@ -262,6 +267,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {

267 worker_add_job(wg, keyword, transaction, function, timeout_s, NULL, access, source);

268 }

269 else if(keyword && (strcmp(keyword, PLUGINSD_CALL_FUNCTION_PAYLOAD_BEGIN) == 0)) {

270 + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION PAYLOAD CALL");

271 char *transaction = get_word(words, num_words, 1);

272 char *timeout_s = get_word(words, num_words, 2);

273 char *function = get_word(words, num_words, 3);

@@ -279,6 +285,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {

285 deferred.enabled = true;

286 }

287 else if(keyword && strcmp(keyword, PLUGINSD_CALL_FUNCTION_CANCEL) == 0) {

288 + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION CANCEL");

289 char *transaction = get_word(words, num_words, 1);

290 const DICTIONARY_ITEM *acquired = dictionary_get_and_acquire_item(wg->worker_queue, transaction);

291 if(acquired) {

@@ -292,6 +299,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {

299 nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received CANCEL for transaction '%s', but it not available here", transaction);

300 }

301 else if(keyword && strcmp(keyword, PLUGINSD_CALL_FUNCTION_PROGRESS) == 0) {

302 + // nd_log(NDLS_COLLECTORS, NDLP_INFO, "FUNCTION PROGRESS");

303 char *transaction = get_word(words, num_words, 1);

304 const DICTIONARY_ITEM *acquired = dictionary_get_and_acquire_item(wg->worker_queue, transaction);

305 if(acquired) {

@@ -305,7 +313,7 @@ static void *rrd_functions_worker_globals_reader_main(void *arg) {

313 nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received PROGRESS for transaction '%s', but it not available here", transaction);

314 }

315 else

308 - nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received unknown command: %s", keyword?keyword:"(unset)");

316 + nd_log(NDLS_COLLECTORS, NDLP_NOTICE, "Received unknown command: %s", keyword ? keyword : "(unset)");

317

318 buffer_flush(buffer);

319 }

src/web/api/v1/api_v1_function.c

+1 -1

@@ -39,6 +39,6 @@ int api_v1_function(RRDHOST *host, struct web_client *w, char *url) {

39 return rrd_function_run(host, wb, timeout, w->access, function, true, transaction,

40 NULL, NULL,

41 web_client_progress_functions_update, w,

42 - web_client_interrupt_callback, w, NULL,

42 + web_client_interrupt_callback, w, w->payload,

43 buffer_tostring(source), false);

44 }