@cryptotaxi247 / netdata-1 / commits / 51decad98

Add system info streaming (#5996)

* Add system info streaming * Fix segmentation fault in unit testing

Vladimir Kobal committed May 13, 2019 at 15:12 UTC 51decad9896131ed478681d90886d63bbd953b14

5 files changed +193 -27

daemon/main.c

+16 -9

@@ -650,7 +650,7 @@ static int load_netdata_conf(char *filename, char overwrite_used) {

650 return ret;

651 }

652

653 -int get_system_info () {

653 +int get_system_info(struct rrdhost_system_info *system_info) {

654 char *script;

655 script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("system-info.sh") + 2));

656 sprintf(script, "%s/%s", netdata_configured_primary_plugins_dir, "system-info.sh");

@@ -679,10 +679,15 @@ int get_system_info () {

679 (*newline) = '\0';

680 }

681 char n[51], v[101];

682 - snprintfz(n,50,"%s",name);

683 - snprintfz(v,101,"%s",value);

684 - info("%s=%s", n, v);

685 - setenv(n, v, 1);

682 + snprintfz(n, 50,"%s",name);

683 + snprintfz(v, 101,"%s",value);

684 + if(unlikely(rrdhost_set_system_info_variable(system_info, n, v))) {

685 + info("Unexpected environment variable %s=%s", n, v);

686 + }

687 + else {

688 + info("%s=%s", n, v);

689 + setenv(n, v, 1);

690 + }

691 }

692 }

693 mypclose(fp, command_pid);

@@ -844,7 +849,7 @@ int main(int argc, char **argv) {

849 default_rrd_update_every = 1;

850 default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;

851 default_health_enabled = 0;

847 - rrd_init("unittest");

852 + rrd_init("unittest", NULL);

853 default_rrdpush_enabled = 0;

854 if(run_all_mockup_tests()) return 1;

855 if(unit_test_storage()) return 1;

@@ -1093,8 +1098,10 @@ int main(int argc, char **argv) {

1098

1099 // initialize the log files

1100 open_all_log_files();

1101 +

1102 netdata_anonymous_statistics_enabled=-1;

1097 - if (get_system_info() == 0) send_statistics("START","-", "-");

1103 + struct rrdhost_system_info *system_info = calloc(1, sizeof(struct rrdhost_system_info));

1104 + if (get_system_info(system_info) == 0) send_statistics("START","-", "-");

1105

1106 #ifdef NETDATA_INTERNAL_CHECKS

1107 if(debug_flags != 0) {

@@ -1129,14 +1136,14 @@ int main(int argc, char **argv) {

1136 // ------------------------------------------------------------------------

1137 // initialize rrd, registry, health, rrdpush, etc.

1138

1132 - rrd_init(netdata_configured_hostname);

1139 + rrd_init(netdata_configured_hostname, system_info);

1140 + rrdhost_system_info_free(system_info);

1141

1142 // ------------------------------------------------------------------------

1143 // enable log flood protection

1144

1145 error_log_limit_reset();

1146

1139 -

1147 // ------------------------------------------------------------------------

1148 // spawn the threads

1149

database/rrd.h

+24 -1

@@ -490,6 +490,22 @@ typedef struct alarm_log {

490 // ----------------------------------------------------------------------------

491 // RRD HOST

492

493 +struct rrdhost_system_info {

494 + char *os_name;

495 + char *os_id;

496 + char *os_id_like;

497 + char *os_version;

498 + char *os_version_id;

499 + char *os_detection;

500 + char *kernel_name;

501 + char *kernel_version;

502 + char *architecture;

503 + char *virtualization;

504 + char *virt_detection;

505 + char *container;

506 + char *container_detection;

507 +};

508 +

509 struct rrdhost {

510 avl avl; // the index of hosts

511

@@ -520,6 +536,8 @@ struct rrdhost {

536 char *program_name; // the program name that collects metrics for this host

537 char *program_version; // the program version that collects metrics for this host

538

539 + struct rrdhost_system_info *system_info; // information collected from the host environment

540 +

541 // ------------------------------------------------------------------------

542 // streaming of data to remote hosts - rrdpush

543

@@ -634,7 +652,7 @@ extern netdata_rwlock_t rrd_rwlock;

652 extern size_t rrd_hosts_available;

653 extern time_t rrdhost_free_orphan_time;

654

637 -extern void rrd_init(char *hostname);

655 +extern void rrd_init(char *hostname, struct rrdhost_system_info *system_info);

656

657 extern RRDHOST *rrdhost_find_by_hostname(const char *hostname, uint32_t hash);

658 extern RRDHOST *rrdhost_find_by_guid(const char *guid, uint32_t hash);

@@ -656,8 +674,12 @@ extern RRDHOST *rrdhost_find_or_create(

674 , char *rrdpush_destination

675 , char *rrdpush_api_key

676 , char *rrdpush_send_charts_matching

677 + , struct rrdhost_system_info *system_info

678 );

679

680 +extern int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, char *name, char *value);

681 +extern struct rrdhost_system_info *rrdhost_system_info_dup(struct rrdhost_system_info *system_info);

682 +

683 #if defined(NETDATA_INTERNAL_CHECKS) && defined(NETDATA_VERIFY_LOCKS)

684 extern void __rrdhost_check_wrlock(RRDHOST *host, const char *file, const char *function, const unsigned long line);

685 extern void __rrdhost_check_rdlock(RRDHOST *host, const char *file, const char *function, const unsigned long line);

@@ -714,6 +736,7 @@ extern void rrdhost_save_all(void);

736 extern void rrdhost_cleanup_all(void);

737

738 extern void rrdhost_cleanup_orphan_hosts_nolock(RRDHOST *protected);

739 +extern void rrdhost_system_info_free(struct rrdhost_system_info *system_info);

740 extern void rrdhost_free(RRDHOST *host);

741 extern void rrdhost_save_charts(RRDHOST *host);

742 extern void rrdhost_delete_charts(RRDHOST *host);

database/rrdhost.c

+99 -1

@@ -122,6 +122,7 @@ RRDHOST *rrdhost_create(const char *hostname,

122 char *rrdpush_destination,

123 char *rrdpush_api_key,

124 char *rrdpush_send_charts_matching,

125 + struct rrdhost_system_info *system_info,

126 int is_localhost

127 ) {

128 debug(D_RRDHOST, "Host '%s': adding with guid '%s'", hostname, guid);

@@ -157,6 +158,8 @@ RRDHOST *rrdhost_create(const char *hostname,

158 host->program_version = strdupz((program_version && *program_version)?program_version:"unknown");

159 host->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname);

160

161 + host->system_info = rrdhost_system_info_dup(system_info);

162 +

163 avl_init_lock(&(host->rrdset_root_index), rrdset_compare);

164 avl_init_lock(&(host->rrdset_root_index_name), rrdset_compare_name);

165 avl_init_lock(&(host->rrdfamily_root_index), rrdfamily_compare);

@@ -332,6 +335,7 @@ RRDHOST *rrdhost_find_or_create(

335 , char *rrdpush_destination

336 , char *rrdpush_api_key

337 , char *rrdpush_send_charts_matching

338 + , struct rrdhost_system_info *system_info

339 ) {

340 debug(D_RRDHOST, "Searching for host '%s' with guid '%s'", hostname, guid);

341

@@ -355,6 +359,7 @@ RRDHOST *rrdhost_find_or_create(

359 , rrdpush_destination

360 , rrdpush_api_key

361 , rrdpush_send_charts_matching

362 + , system_info

363 , 0

364 );

365 }

@@ -439,7 +444,7 @@ restart_after_removal:

444 // ----------------------------------------------------------------------------

445 // RRDHOST global / startup initialization

446

442 -void rrd_init(char *hostname) {

447 +void rrd_init(char *hostname, struct rrdhost_system_info *system_info) {

448 rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds", rrdset_free_obsolete_time);

449 gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_GLOBAL, "gap when lost iterations above", gap_when_lost_iterations_above);

450 if (gap_when_lost_iterations_above < 1)

@@ -468,6 +473,7 @@ void rrd_init(char *hostname) {

473 , default_rrdpush_destination

474 , default_rrdpush_api_key

475 , default_rrdpush_send_charts_matching

476 + , system_info

477 , 1

478 );

479 rrd_unlock();

@@ -513,6 +519,27 @@ void __rrd_check_wrlock(const char *file, const char *function, const unsigned l

519 // ----------------------------------------------------------------------------

520 // RRDHOST - free

521

522 +void rrdhost_system_info_free(struct rrdhost_system_info *system_info) {

523 + info("SYSTEM_INFO: free %p", system_info);

524 +

525 + if(likely(system_info)) {

526 + freez(system_info->os_name);

527 + freez(system_info->os_id);

528 + freez(system_info->os_id_like);

529 + freez(system_info->os_version);

530 + freez(system_info->os_version_id);

531 + freez(system_info->os_detection);

532 + freez(system_info->kernel_name);

533 + freez(system_info->kernel_version);

534 + freez(system_info->architecture);

535 + freez(system_info->virtualization);

536 + freez(system_info->virt_detection);

537 + freez(system_info->container);

538 + freez(system_info->container_detection);

539 + freez(system_info);

540 + }

541 +}

542 +

543 void rrdhost_free(RRDHOST *host) {

544 if(!host) return;

545

@@ -573,6 +600,7 @@ void rrdhost_free(RRDHOST *host) {

600 freez((void *)host->timezone);

601 freez(host->program_version);

602 freez(host->program_name);

603 + rrdhost_system_info_free(host->system_info);

604 freez(host->cache_dir);

605 freez(host->varlib_dir);

606 freez(host->rrdpush_send_api_key);

@@ -744,3 +772,73 @@ restart_after_removal:

772 }

773 }

774 }

775 +

776 +// ----------------------------------------------------------------------------

777 +// RRDHOST - set system info from environment variables

778 +

779 +int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, char *name, char *value) {

780 + if(!strcmp(name, "NETDATA_SYSTEM_OS_NAME")){

781 + system_info->os_name = strdupz(value);

782 + }

783 + else if(!strcmp(name, "NETDATA_SYSTEM_OS_ID")){

784 + system_info->os_id = strdupz(value);

785 + }

786 + else if(!strcmp(name, "NETDATA_SYSTEM_OS_ID_LIKE")){

787 + system_info->os_id_like = strdupz(value);

788 + }

789 + else if(!strcmp(name, "NETDATA_SYSTEM_OS_VERSION")){

790 + system_info->os_version = strdupz(value);

791 + }

792 + else if(!strcmp(name, "NETDATA_SYSTEM_OS_VERSION_ID")){

793 + system_info->os_version_id = strdupz(value);

794 + }

795 + else if(!strcmp(name, "NETDATA_SYSTEM_OS_DETECTION")){

796 + system_info->os_detection = strdupz(value);

797 + }

798 + else if(!strcmp(name, "NETDATA_SYSTEM_KERNEL_NAME")){

799 + system_info->kernel_name = strdupz(value);

800 + }

801 + else if(!strcmp(name, "NETDATA_SYSTEM_KERNEL_VERSION")){

802 + system_info->kernel_version = strdupz(value);

803 + }

804 + else if(!strcmp(name, "NETDATA_SYSTEM_ARCHITECTURE")){

805 + system_info->architecture = strdupz(value);

806 + }

807 + else if(!strcmp(name, "NETDATA_SYSTEM_VIRTUALIZATION")){

808 + system_info->virtualization = strdupz(value);

809 + }

810 + else if(!strcmp(name, "NETDATA_SYSTEM_VIRT_DETECTION")){

811 + system_info->virt_detection = strdupz(value);

812 + }

813 + else if(!strcmp(name, "NETDATA_SYSTEM_CONTAINER")){

814 + system_info->container = strdupz(value);

815 + }

816 + else if(!strcmp(name, "NETDATA_SYSTEM_CONTAINER_DETECTION")){

817 + system_info->container_detection = strdupz(value);

818 + }

819 + else return 1;

820 +

821 + return 0;

822 +}

823 +

824 +struct rrdhost_system_info *rrdhost_system_info_dup(struct rrdhost_system_info *system_info) {

825 + struct rrdhost_system_info *ret = callocz(1, sizeof(struct rrdhost_system_info));

826 +

827 + if(likely(system_info)) {

828 + ret->os_name = strdupz(system_info->os_name);

829 + ret->os_id = strdupz(system_info->os_id);

830 + ret->os_id_like = strdupz(system_info->os_id_like);

831 + ret->os_version = strdupz(system_info->os_version);

832 + ret->os_version_id = strdupz(system_info->os_version_id);

833 + ret->os_detection = strdupz(system_info->os_detection);

834 + ret->kernel_name = strdupz(system_info->kernel_name);

835 + ret->kernel_version = strdupz(system_info->kernel_version);

836 + ret->architecture = strdupz(system_info->architecture);

837 + ret->virtualization = strdupz(system_info->virtualization);

838 + ret->virt_detection = strdupz(system_info->virt_detection);

839 + ret->container = strdupz(system_info->container);

840 + ret->container_detection = strdupz(system_info->container_detection);

841 + }

842 +

843 + return ret;

844 +}

streaming/rrdpush.c

+40 -2

@@ -445,7 +445,21 @@ static int rrdpush_sender_thread_connect_to_master(RRDHOST *host, int default_po

445 #define HTTP_HEADER_SIZE 8192

446 char http[HTTP_HEADER_SIZE + 1];

447 snprintfz(http, HTTP_HEADER_SIZE,

448 - "STREAM key=%s&hostname=%s&registry_hostname=%s&machine_guid=%s&update_every=%d&os=%s&timezone=%s&tags=%s HTTP/1.1\r\n"

448 + "STREAM key=%s&hostname=%s&registry_hostname=%s&machine_guid=%s&update_every=%d&os=%s&timezone=%s&tags=%s"

449 + "&NETDATA_SYSTEM_OS_NAME=%s"

450 + "&NETDATA_SYSTEM_OS_ID=%s"

451 + "&NETDATA_SYSTEM_OS_ID_LIKE=%s"

452 + "&NETDATA_SYSTEM_OS_VERSION=%s"

453 + "&NETDATA_SYSTEM_OS_VERSION_ID=%s"

454 + "&NETDATA_SYSTEM_OS_DETECTION=%s"

455 + "&NETDATA_SYSTEM_KERNEL_NAME=%s"

456 + "&NETDATA_SYSTEM_KERNEL_VERSION=%s"

457 + "&NETDATA_SYSTEM_ARCHITECTURE=%s"

458 + "&NETDATA_SYSTEM_VIRTUALIZATION=%s"

459 + "&NETDATA_SYSTEM_VIRT_DETECTION=%s"

460 + "&NETDATA_SYSTEM_CONTAINER=%s"

461 + "&NETDATA_SYSTEM_CONTAINER_DETECTION=%s"

462 + " HTTP/1.1\r\n"

463 "User-Agent: %s/%s\r\n"

464 "Accept: */*\r\n\r\n"

465 , host->rrdpush_send_api_key

@@ -456,6 +470,19 @@ static int rrdpush_sender_thread_connect_to_master(RRDHOST *host, int default_po

470 , host->os

471 , host->timezone

472 , (host->tags)?host->tags:""

473 + , host->system_info->os_name

474 + , host->system_info->os_id

475 + , host->system_info->os_id_like

476 + , host->system_info->os_version

477 + , host->system_info->os_version_id

478 + , host->system_info->os_detection

479 + , host->system_info->kernel_name

480 + , host->system_info->kernel_version

481 + , host->system_info->architecture

482 + , host->system_info->virtualization

483 + , host->system_info->virt_detection

484 + , host->system_info->container

485 + , host->system_info->container_detection

486 , host->program_name

487 , host->program_version

488 );

@@ -821,6 +848,7 @@ static int rrdpush_receive(int fd

848 , const char *tags

849 , const char *program_name

850 , const char *program_version

851 + , struct rrdhost_system_info *system_info

852 , int update_every

853 , char *client_ip

854 , char *client_port

@@ -894,6 +922,7 @@ static int rrdpush_receive(int fd

922 , rrdpush_destination

923 , rrdpush_api_key

924 , rrdpush_send_charts_matching

925 + , system_info

926 );

927

928 if(!host) {

@@ -1027,6 +1056,7 @@ struct rrdpush_thread {

1056 char *client_port;

1057 char *program_name;

1058 char *program_version;

1059 + struct rrdhost_system_info *system_info;

1060 int update_every;

1061 };

1062

@@ -1049,6 +1079,7 @@ static void rrdpush_receiver_thread_cleanup(void *ptr) {

1079 freez(rpt->client_port);

1080 freez(rpt->program_name);

1081 freez(rpt->program_version);

1082 + rrdhost_system_info_free(rpt->system_info);

1083 freez(rpt);

1084 }

1085 }

@@ -1070,6 +1101,7 @@ static void *rrdpush_receiver_thread(void *ptr) {

1101 , rpt->tags

1102 , rpt->program_name

1103 , rpt->program_version

1104 + , rpt->system_info

1105 , rpt->update_every

1106 , rpt->client_ip

1107 , rpt->client_port

@@ -1120,6 +1152,8 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url

1152 int update_every = default_rrd_update_every;

1153 char buf[GUID_LEN + 1];

1154

1155 + struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info));

1156 +

1157 while(url) {

1158 char *value = mystrsep(&url, "&");

1159 if(!value || !*value) continue;

@@ -1145,7 +1179,9 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url

1179 else if(!strcmp(name, "tags"))

1180 tags = value;

1181 else

1148 - info("STREAM [receive from [%s]:%s]: request has parameter '%s' = '%s', which is not used.", w->client_ip, w->client_port, key, value);

1182 + if(unlikely(rrdhost_set_system_info_variable(system_info, name, value))) {

1183 + info("STREAM [receive from [%s]:%s]: request has parameter '%s' = '%s', which is not used.", w->client_ip, w->client_port, key, value);

1184 + }

1185 }

1186

1187 if(!key || !*key) {

@@ -1248,6 +1284,7 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url

1284 rpt->client_ip = strdupz(w->client_ip);

1285 rpt->client_port = strdupz(w->client_port);

1286 rpt->update_every = update_every;

1287 + rpt->system_info = system_info;

1288

1289 if(w->user_agent && w->user_agent[0]) {

1290 char *t = strchr(w->user_agent, '/');

@@ -1260,6 +1297,7 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url

1297 if(t && *t) rpt->program_version = strdupz(t);

1298 }

1299

1300 +

1301 netdata_thread_t thread;

1302

1303 debug(D_SYSTEM, "starting STREAM receive thread.");

web/api/web_api_v1.c

+14 -14

@@ -736,20 +736,20 @@ inline int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, c

736 web_client_api_request_v1_info_summary_alarm_statuses(host, wb);

737 buffer_strcat(wb, "\t},\n");

738

739 - buffer_sprintf(wb, "\t\"os_name\": %s,\n", getenv("NETDATA_SYSTEM_OS_NAME"));

740 - buffer_sprintf(wb, "\t\"os_id\": \"%s\",\n", getenv("NETDATA_SYSTEM_OS_ID"));

741 - buffer_sprintf(wb, "\t\"os_id_like\": \"%s\",\n", getenv("NETDATA_SYSTEM_OS_ID_LIKE"));

742 - buffer_sprintf(wb, "\t\"os_version\": \"%s\",\n", getenv("NETDATA_SYSTEM_OS_VERSION"));

743 - buffer_sprintf(wb, "\t\"os_version_id\": \"%s\",\n", getenv("NETDATA_SYSTEM_OS_VERSION_ID"));

744 - buffer_sprintf(wb, "\t\"os_detection\": \"%s\",\n", getenv("NETDATA_SYSTEM_OS_DETECTION"));

745 - buffer_sprintf(wb, "\t\"kernel_name\": \"%s\",\n", getenv("NETDATA_SYSTEM_KERNEL_NAME"));

746 - buffer_sprintf(wb, "\t\"kernel_version\": \"%s\",\n", getenv("NETDATA_SYSTEM_KERNEL_VERSION"));

747 - buffer_sprintf(wb, "\t\"architecture\": \"%s\",\n", getenv("NETDATA_SYSTEM_ARCHITECTURE"));

748 - buffer_sprintf(wb, "\t\"virtualization\": \"%s\",\n", getenv("NETDATA_SYSTEM_VIRTUALIZATION"));

749 - buffer_sprintf(wb, "\t\"virt_detection\": \"%s\",\n", getenv("NETDATA_SYSTEM_VIRT_DETECTION"));

750 - buffer_sprintf(wb, "\t\"container\": \"%s\",\n", getenv("NETDATA_SYSTEM_CONTAINER"));

751 - buffer_sprintf(wb, "\t\"container_detection\": \"%s\",\n", getenv("NETDATA_SYSTEM_CONTAINER_DETECTION"));

752 -

739 + buffer_sprintf(wb, "\t\"os_name\": %s,\n", host->system_info->os_name);

740 + buffer_sprintf(wb, "\t\"os_id\": \"%s\",\n", host->system_info->os_id);

741 + buffer_sprintf(wb, "\t\"os_id_like\": \"%s\",\n", host->system_info->os_id_like);

742 + buffer_sprintf(wb, "\t\"os_version\": \"%s\",\n", host->system_info->os_version);

743 + buffer_sprintf(wb, "\t\"os_version_id\": \"%s\",\n", host->system_info->os_version_id);

744 + buffer_sprintf(wb, "\t\"os_detection\": \"%s\",\n", host->system_info->os_detection);

745 + buffer_sprintf(wb, "\t\"kernel_name\": \"%s\",\n", host->system_info->kernel_name);

746 + buffer_sprintf(wb, "\t\"kernel_version\": \"%s\",\n", host->system_info->kernel_version);

747 + buffer_sprintf(wb, "\t\"architecture\": \"%s\",\n", host->system_info->architecture);

748 + buffer_sprintf(wb, "\t\"virtualization\": \"%s\",\n", host->system_info->virtualization);

749 + buffer_sprintf(wb, "\t\"virt_detection\": \"%s\",\n", host->system_info->virt_detection);

750 + buffer_sprintf(wb, "\t\"container\": \"%s\",\n", host->system_info->container);

751 + buffer_sprintf(wb, "\t\"container_detection\": \"%s\",\n", host->system_info->container_detection);

752 +

753 buffer_strcat(wb, "\t\"collectors\": [");

754 chartcollectors2json(host, wb);

755 buffer_strcat(wb, "\n\t]\n");