@cryptotaxi247 / netdata-1 / commits / 0882ed03b

Add snapshot message and calls to sql_queue_removed_alerts_to_aclk (#11664)

Emmanuel Vasilakis committed Oct 19, 2021 at 11:30 UTC 0882ed03b4000b6f9e1f64743321e4cd6e2aa39f

8 files changed +278 -1

aclk/aclk.c

@@ -25,6 +25,8 @@

26 int aclk_pubacks_per_conn = 0; // How many PubAcks we got since MQTT conn est.

28 +int aclk_alert_reloaded = 1; //1 on startup, and again on health_reload

29 +

30 time_t aclk_block_until = 0;

32 aclk_env_t *aclk_env = NULL;

aclk/aclk_api.h

@@ -18,6 +18,7 @@ extern int aclk_disable_runtime;

18 extern int aclk_disable_single_updates;

20 extern int aclk_stats_enabled;

21 +extern int aclk_alert_reloaded;

23 extern int aclk_ng;

aclk/aclk_rx_msgs.c

+11

@@ -398,6 +398,17 @@ void aclk_handle_new_cloud_msg(const char *message_type, const char *msg, size_t

398 freez(config_hash);

399 return;

400 }

401 + if (!strcmp(message_type, "SendAlarmSnapshot")) {

402 + struct send_alarm_snapshot *sas = parse_send_alarm_snapshot(msg, msg_len);

403 + if (!sas->node_id || !sas->claim_id) {

404 + error("Error parsing SendAlarmSnapshot");

405 + destroy_send_alarm_snapshot(sas);

406 + return;

407 + }

408 + aclk_process_send_alarm_snapshot(sas->node_id, sas->claim_id, sas->snapshot_id, sas->sequence_id);

409 + destroy_send_alarm_snapshot(sas);

410 + return;

411 + }

412 error ("Unknown new cloud arch message type received \"%s\"", message_type);

413 }

414 #endif

database/sqlite/sqlite_aclk.c

@@ -412,6 +412,10 @@ void aclk_database_worker(void *arg)

412 debug(D_ACLK_SYNC, "Pushing alarm health log to the cloud for %s", wc->host_guid);

413 aclk_push_alarm_health_log(wc, cmd);

414 break;

415 + case ACLK_DATABASE_PUSH_ALERT_SNAPSHOT:

416 + debug(D_ACLK_SYNC, "Pushing alert snapshot to the cloud for node %s", wc->host_guid);

417 + aclk_push_alert_snapshot_event(wc, cmd);

418 + break;

419

420 // NODE OPERATIONS

421 case ACLK_DATABASE_NODE_INFO:

database/sqlite/sqlite_aclk.h

@@ -124,6 +124,7 @@ enum aclk_database_opcode {

124 ACLK_DATABASE_NODE_INFO,

125 ACLK_DATABASE_PUSH_ALERT,

126 ACLK_DATABASE_PUSH_ALERT_CONFIG,

127 + ACLK_DATABASE_PUSH_ALERT_SNAPSHOT,

128 ACLK_DATABASE_PUSH_CHART,

129 ACLK_DATABASE_PUSH_CHART_CONFIG,

130 ACLK_DATABASE_RESET_CHART,

@@ -170,6 +171,8 @@ struct aclk_database_worker_config {

171 uint64_t alerts_batch_id; // batch id for alerts to use

172 uint64_t alerts_start_seq_id; // cloud has asked to start streaming from

173 uint64_t alert_sequence_id; // last alert sequence_id

174 + uint64_t alerts_snapshot_id; //will contain the snapshot_id value if snapshot was requested

175 + uint64_t alerts_ack_sequence_id; //last sequence_id ack'ed from cloud via sendsnapshot message

176 uv_loop_t *loop;

177 RRDHOST *host;

178 uv_async_t async;

database/sqlite/sqlite_aclk_alert.c

+245 -1

@@ -147,7 +147,11 @@ void aclk_push_alert_event(struct aclk_database_worker_config *wc, struct aclk_d

147 sql,

148 "UPDATE aclk_alert_%s SET date_submitted = NULL, date_cloud_ack = NULL WHERE sequence_id >= %"PRIu64

149 "; UPDATE aclk_alert_%s SET date_cloud_ack = strftime('%%s','now') WHERE sequence_id < %"PRIu64

150 - " and date_cloud_ack is null",

150 + " and date_cloud_ack is null "

151 + "; UPDATE aclk_alert_%s SET date_submitted = strftime('%%s','now') WHERE sequence_id < %"PRIu64

152 + " and date_submitted is null",

153 + wc->uuid_str,

154 + wc->alerts_start_seq_id,

155 wc->uuid_str,

156 wc->alerts_start_seq_id,

157 wc->uuid_str,

@@ -556,6 +560,11 @@ void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start

560

561 int sql_queue_removed_alerts_to_aclk(RRDHOST *host)

562 {

563 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL

564 + if (!aclk_use_new_cloud_arch) {

565 + return 0;

566 + }

567 +

568 CHECK_SQLITE_CONNECTION(db_meta);

569

570 struct aclk_database_worker_config *wc = (struct aclk_database_worker_config *) host->dbsync_worker;

@@ -574,6 +583,241 @@ int sql_queue_removed_alerts_to_aclk(RRDHOST *host)

583 db_execute(buffer_tostring(sql));

584

585 buffer_free(sql);

586 +#else

587 + UNUSED(host);

588 +#endif

589 + return 0;

590 +}

591 +

592 +void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id, uint64_t snapshot_id, uint64_t sequence_id)

593 +{

594 + UNUSED(claim_id);

595 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL

596 + if (unlikely(!node_id))

597 + return;

598 +

599 + uuid_t node_uuid;

600 + if (uuid_parse(node_id, node_uuid))

601 + return;

602 +

603 + struct aclk_database_worker_config *wc = NULL;

604 + rrd_wrlock();

605 + RRDHOST *host = find_host_by_node_id(node_id);

606 + if (likely(host))

607 + wc = (struct aclk_database_worker_config *)host->dbsync_worker;

608 + rrd_unlock();

609 +

610 + if (likely(wc)) {

611 + info(

612 + "Send alerts snapshot requested for %s with snapshot_id %" PRIu64 " and ack sequence_id %" PRIu64,

613 + node_id,

614 + snapshot_id,

615 + sequence_id);

616 + __sync_synchronize();

617 + wc->alerts_snapshot_id = snapshot_id;

618 + wc->alerts_ack_sequence_id = sequence_id;

619 + __sync_synchronize();

620 +

621 + struct aclk_database_cmd cmd;

622 + memset(&cmd, 0, sizeof(cmd));

623 + cmd.opcode = ACLK_DATABASE_PUSH_ALERT_SNAPSHOT;

624 + cmd.data_param = NULL;

625 + cmd.completion = NULL;

626 + aclk_database_enq_cmd(wc, &cmd);

627 + } else

628 + error("ACLK synchronization thread is not active for host %s", host->hostname);

629 +#else

630 + UNUSED(node_id);

631 + UNUSED(snapshot_id);

632 + UNUSED(sequence_id);

633 +#endif

634 + return;

635 +}

636 +

637 +void aclk_mark_alert_cloud_ack(char *uuid_str, uint64_t alerts_ack_sequence_id)

638 +{

639 + BUFFER *sql = buffer_create(1024);

640 +

641 + if (alerts_ack_sequence_id != 0) {

642 + buffer_sprintf(

643 + sql,

644 + "UPDATE aclk_alert_%s SET date_cloud_ack = strftime('%%s','now') WHERE sequence_id <= %" PRIu64 "",

645 + uuid_str,

646 + alerts_ack_sequence_id);

647 + db_execute(buffer_tostring(sql));

648 + }

649 +

650 + buffer_free(sql);

651 +}

652 +

653 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL

654 +void health_alarm_entry2proto_nolock(struct alarm_log_entry *alarm_log, ALARM_ENTRY *ae, RRDHOST *host)

655 +{

656 + char *edit_command = ae->source ? health_edit_command_from_source(ae->source) : strdupz("UNKNOWN=0");

657 + char config_hash_id[GUID_LEN + 1];

658 + uuid_unparse_lower(ae->config_hash_id, config_hash_id);

659 +

660 + alarm_log->chart = strdupz((char *)ae->chart);

661 + alarm_log->name = strdupz((char *)ae->name);

662 + alarm_log->family = strdupz((char *)ae->family);

663 +

664 + alarm_log->batch_id = 0;

665 + alarm_log->sequence_id = 0;

666 + alarm_log->when = (time_t)ae->when;

667 +

668 + alarm_log->config_hash = strdupz((char *)config_hash_id);

669 +

670 + alarm_log->utc_offset = host->utc_offset;

671 + alarm_log->timezone = strdupz((char *)host->abbrev_timezone);

672 + alarm_log->exec_path = ae->exec ? strdupz((char *)ae->exec) : strdupz((char *)host->health_default_exec);

673 + alarm_log->conf_source = strdupz(ae->source);

674 +

675 + alarm_log->command = strdupz(edit_command);

676 +

677 + alarm_log->duration = (time_t)ae->duration;

678 + alarm_log->non_clear_duration = (time_t)ae->non_clear_duration;

679 + alarm_log->status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS)ae->new_status);

680 + alarm_log->old_status = rrdcalc_status_to_proto_enum((RRDCALC_STATUS)ae->old_status);

681 + alarm_log->delay = (int)ae->delay;

682 + alarm_log->delay_up_to_timestamp = (time_t)ae->delay_up_to_timestamp;

683 + alarm_log->last_repeat = (time_t)ae->last_repeat;

684 +

685 + alarm_log->silenced =

686 + ((ae->flags & HEALTH_ENTRY_FLAG_SILENCED) || (ae->recipient && !strncmp((char *)ae->recipient, "silent", 6))) ?

687 + 1 :

688 + 0;

689 +

690 + alarm_log->value_string = strdupz(ae->new_value_string);

691 + alarm_log->old_value_string = strdupz(ae->old_value_string);

692 +

693 + alarm_log->value = (!isnan(ae->new_value)) ? (calculated_number)ae->new_value : 0;

694 + alarm_log->old_value = (!isnan(ae->old_value)) ? (calculated_number)ae->old_value : 0;

695 +

696 + alarm_log->updated = (ae->flags & HEALTH_ENTRY_FLAG_UPDATED) ? 1 : 0;

697 + alarm_log->rendered_info = strdupz(ae->info);

698 +

699 + freez(edit_command);

700 +}

701 +#endif

702 +

703 +static int have_recent_alarm(RRDHOST *host, uint32_t alarm_id, time_t mark)

704 +{

705 + ALARM_ENTRY *ae = host->health_log.alarms;

706 +

707 + while (ae) {

708 + if (ae->alarm_id == alarm_id && ae->unique_id > mark &&

709 + (ae->new_status != RRDCALC_STATUS_WARNING && ae->new_status != RRDCALC_STATUS_CRITICAL))

710 + return 1;

711 + ae = ae->next;

712 + }

713

714 return 0;

715 }

716 +

717 +#define ALARM_EVENTS_PER_CHUNK 10

718 +void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd)

719 +{

720 +#ifndef ENABLE_NEW_CLOUD_PROTOCOL

721 + UNUSED(wc);

722 + UNUSED(cmd);

723 +#else

724 + UNUSED(cmd);

725 + // we perhaps we don't need this for snapshots

726 + if (unlikely(!wc->alert_updates)) {

727 + debug(D_ACLK_SYNC, "Ignoring alert push snapshot event, updates have been turned off for node %s", wc->node_id);

728 + return;

729 + }

730 +

731 + char *claim_id = is_agent_claimed();

732 + if (unlikely(!claim_id))

733 + return;

734 +

735 + aclk_mark_alert_cloud_ack(wc->uuid_str, wc->alerts_ack_sequence_id);

736 +

737 + RRDHOST *host = wc->host;

738 + uint32_t cnt = 0;

739 +

740 + netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);

741 +

742 + ALARM_ENTRY *ae = host->health_log.alarms;

743 +

744 + for (; ae; ae = ae->next) {

745 + if (likely(ae->updated_by_id))

746 + continue;

747 +

748 + if (unlikely(ae->new_status == RRDCALC_STATUS_UNINITIALIZED))

749 + continue;

750 +

751 + if (have_recent_alarm(host, ae->alarm_id, ae->unique_id))

752 + continue;

753 +

754 + cnt++;

755 + }

756 +

757 + if (cnt) {

758 + uint32_t chunk = 1, chunks = 0;

759 +

760 + chunks = (cnt / ALARM_EVENTS_PER_CHUNK) + (cnt % ALARM_EVENTS_PER_CHUNK != 0);

761 + ae = host->health_log.alarms;

762 +

763 + cnt = 0;

764 + struct alarm_snapshot alarm_snap;

765 + alarm_snap.node_id = wc->node_id;

766 + alarm_snap.claim_id = claim_id;

767 + alarm_snap.snapshot_id = wc->alerts_snapshot_id;

768 + alarm_snap.chunks = chunks;

769 + alarm_snap.chunk = chunk;

770 +

771 + alarm_snapshot_proto_ptr_t snapshot_proto;

772 + snapshot_proto = generate_alarm_snapshot_proto(&alarm_snap);

773 +

774 + for (; ae; ae = ae->next) {

775 + if (likely(ae->updated_by_id))

776 + continue;

777 +

778 + if (unlikely(ae->new_status == RRDCALC_STATUS_UNINITIALIZED))

779 + continue;

780 +

781 + if (have_recent_alarm(host, ae->alarm_id, ae->unique_id))

782 + continue;

783 +

784 + struct alarm_log_entry alarm_log;

785 + alarm_log.node_id = wc->node_id;

786 + alarm_log.claim_id = claim_id;

787 +

788 + health_alarm_entry2proto_nolock(&alarm_log, ae, host);

789 + add_alarm_log_entry2snapshot(snapshot_proto, &alarm_log);

790 +

791 + cnt++;

792 +

793 + if (cnt == ALARM_EVENTS_PER_CHUNK) {

794 + aclk_send_alarm_snapshot(snapshot_proto);

795 +

796 + cnt = 0;

797 +

798 + if (chunk < chunks) {

799 + chunk++;

800 +

801 + struct alarm_snapshot alarm_snap;

802 + alarm_snap.node_id = wc->node_id;

803 + alarm_snap.claim_id = claim_id;

804 + alarm_snap.snapshot_id = wc->alerts_snapshot_id;

805 + alarm_snap.chunks = chunks;

806 + alarm_snap.chunk = chunk;

807 +

808 + snapshot_proto = generate_alarm_snapshot_proto(&alarm_snap);

809 + }

810 + }

811 + destroy_alarm_log_entry(&alarm_log);

812 + }

813 + if (cnt)

814 + aclk_send_alarm_snapshot(snapshot_proto);

815 + }

816 +

817 + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);

818 + wc->alerts_snapshot_id = 0;

819 +

820 + freez(claim_id);

821 +#endif

822 + return;

823 +}

database/sqlite/sqlite_aclk_alert.h

@@ -13,5 +13,7 @@ void aclk_send_alarm_configuration (char *config_hash);

13 int aclk_push_alert_config_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);

14 void aclk_start_alert_streaming(char *node_id, uint64_t batch_id, uint64_t start_seq_id);

15 int sql_queue_removed_alerts_to_aclk(RRDHOST *host);

16 +void aclk_push_alert_snapshot_event(struct aclk_database_worker_config *wc, struct aclk_database_cmd cmd);

17 +void aclk_process_send_alarm_snapshot(char *node_id, char *claim_id, uint64_t snapshot_id, uint64_t sequence_id);

19 #endif //NETDATA_SQLITE_ACLK_ALERT_H

health/health.c

+10

@@ -230,6 +230,9 @@ void health_reload(void) {

230 if (netdata_cloud_setting) {

231 aclk_single_update_enable();

232 aclk_alarm_reload();

233 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL

234 + aclk_alert_reloaded = 1;

235 +#endif

236 }

237 #endif

238 }

@@ -1035,6 +1038,13 @@ void *health_main(void *ptr) {

1038 rrdhost_unlock(host);

1039 }

1040

1041 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL

1042 + if (aclk_alert_reloaded) {

1043 + sql_queue_removed_alerts_to_aclk(host);

1044 + aclk_alert_reloaded = 0;

1045 + }

1046 +#endif

1047 +

1048 if (unlikely(netdata_exit))

1049 break;

1050