@cryptotaxi247 / netdata-1 / commits / aa3ec552c

Enable support for Netdata Cloud. This PR merges the feature-branch to make the cloud live. It contains the following work: Co-authored-by: Andrew Moss <1043609+amoss@users.noreply.github.com(opens in new tab)> Co-authored-by: Jacek Kolasa <jacek.kolasa@gmail.com(opens in new tab)> Co-authored-by: Austin S. Hemmelgarn <austin@netdata.cloud(opens in new tab)> Co-authored-by: James Mills <prologic@shortcircuit.net.au(opens in new tab)> Co-authored-by: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com(opens in new tab)> Co-authored-by: Timotej S <6674623+underhood@users.noreply.github.com(opens in new tab)> Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com(opens in new tab)> * dashboard with new navbars, v1.0-alpha.9: PR #8478 * dashboard v1.0.11: netdata/dashboard#76 Co-authored-by: Jacek Kolasa <jacek.kolasa@gmail.com(opens in new tab)> * Added installer code to bundle JSON-c if it's not present. PR #8836 Co-authored-by: James Mills <prologic@shortcircuit.net.au(opens in new tab)> * Fix claiming config PR #8843 * Adds JSON-c as hard dep. for ACLK PR #8838 * Fix SSL renegotiation errors in old versions of openssl. PR #8840. Also - we have a transient problem with opensuse CI so this PR disables them with a commit from @prologic. Co-authored-by: James Mills <prologic@shortcircuit.net.au(opens in new tab)> * Fix claiming error handling PR #8850 * Added CI to verify JSON-C bundling code in installer PR #8853 * Make cloud-enabled flag in web/api/v1/info be independent of ACLK build success PR #8866 * Reduce ACLK_STABLE_TIMEOUT from 10 to 3 seconds PR #8871 * remove old-cloud related UI from old dashboard (accessible now via /old suffix) PR #8858 * dashboard v1.0.13 PR #8870 * dashboard v1.0.14 PR #8904 * Provide feedback on proxy setting changes PR #8895 * Change the name of the connect message to update during an ongoing session PR #8927 * Fetch active alarms from alarm_log PR #8944

Andrew Moss committed May 11, 2020 at 08:34 UTC aa3ec552c896aebafd03b9d2c1864272dcb34749

38 files changed +703 -284

.github/dockerfiles/Dockerfile.build_test

@@ -7,5 +7,6 @@ ENV PRE=${PRE}

8 COPY . /netdata

10 +RUN chmod +x /netdata/rmjsonc.sh

11 RUN /bin/sh /netdata/prep-cmd.sh

12 RUN /netdata/packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all

.github/workflows/build-and-install.yml

+39 -2

@@ -9,6 +9,7 @@ jobs:

9 build:

10 name: Build & Install

11 strategy:

12 + fail-fast: false

13 matrix:

14 distro:

15 - 'alpine:edge'

@@ -35,30 +36,59 @@ jobs:

36 include:

37 - distro: 'alpine:edge'

38 pre: 'apk add -U bash'

39 + rmjsonc: 'apk del json-c-dev'

40 - distro: 'alpine:3.11'

41 pre: 'apk add -U bash'

42 + rmjsonc: 'apk del json-c-dev'

43 - distro: 'alpine:3.10'

44 pre: 'apk add -U bash'

45 + rmjsonc: 'apk del json-c-dev'

46 - distro: 'alpine:3.9'

47 pre: 'apk add -U bash'

48 + rmjsonc: 'apk del json-c-dev'

50 - distro: 'archlinux:latest'

51 pre: 'pacman --noconfirm -Sy grep libffi'

53 + - distro: 'centos:8'

54 + rmjsonc: 'dnf remove -y json-c-devel'

55 +

56 - distro: 'debian:bullseye'

57 pre: 'apt-get update'

58 + rmjsonc: 'apt-get remove -y libjson-c-dev'

59 - distro: 'debian:buster'

60 pre: 'apt-get update'

61 + rmjsonc: 'apt-get remove -y libjson-c-dev'

62 - distro: 'debian:stretch'

63 pre: 'apt-get update'

64 + rmjsonc: 'apt-get remove -y libjson-c-dev'

65 +

66 + - distro: 'fedora:32'

67 + rmjsonc: 'dnf remove -y json-c-devel'

68 + - distro: 'fedora:31'

69 + rmjsonc: 'dnf remove -y json-c-devel'

70 + - distro: 'fedora:30'

71 + rmjsonc: 'dnf remove -y json-c-devel'

72 +

73 + - distro: 'opensuse/leap:15.2'

74 + rmjsonc: 'zypper rm -y libjson-c-devel'

75 + - distro: 'opensuse/leap:15.1'

76 + rmjsonc: 'zypper rm -y libjson-c-devel'

77 + - distro: 'opensuse/tumbleweed:latest'

78 + rmjsonc: 'zypper rm -y libjson-c-devel'

79 +

80 - distro: 'ubuntu:20.04'

81 pre: 'apt-get update'

82 + rmjsonc: 'apt-get remove -y libjson-c-dev'

83 - distro: 'ubuntu:19.10'

84 pre: 'apt-get update'

85 + rmjsonc: 'apt-get remove -y libjson-c-dev'

86 - distro: 'ubuntu:18.04'

87 pre: 'apt-get update'

88 + rmjsonc: 'apt-get remove -y libjson-c-dev'

89 - distro: 'ubuntu:16.04'

90 pre: 'apt-get update'

91 + rmjsonc: 'apt-get remove -y libjson-c-dev'

92 runs-on: ubuntu-latest

93 steps:

94 - name: Git clone repository

@@ -66,15 +96,22 @@ jobs:

96 - name: install-required-packages.sh on ${{ matrix.distro }}

97 env:

98 PRE: ${{ matrix.pre }}

99 + RMJSONC: ${{ matrix.rmjsonc }}

100 run: |

101 echo $PRE > ./prep-cmd.sh

102 + echo $RMJSONC > ./rmjsonc.sh

103 docker build . -f .github/dockerfiles/Dockerfile.build_test -t test --build-arg BASE=${{ matrix.distro }}

104 - name: Regular build on ${{ matrix.distro }}

105 run: |

106 docker run -w /netdata test /bin/sh -c 'autoreconf -ivf && ./configure && make -j2'

75 - - name: netdata-installer on ${{ matrix.distro }}

107 + - name: netdata-installer on ${{ matrix.distro }}, disable cloud

108 run: |

109 docker run -w /netdata test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --disable-cloud'

78 - - name: netdata-installer on ${{ matrix.distro }}

110 + - name: netdata-installer on ${{ matrix.distro }}, require cloud

111 run: |

112 docker run -w /netdata test /bin/sh -c './netdata-installer.sh --dont-wait --dont-start-it --require-cloud'

113 + - name: netdata-installer on ${{ matrix.distro }}, require cloud, no JSON-C

114 + if: matrix.rmjsonc != ''

115 + run: |

116 + docker run -w /netdata test \

117 + /bin/sh -c '/netdata/rmjsonc.sh && ./netdata-installer.sh --dont-wait --dont-start-it --require-cloud'

Makefile.am

@@ -624,6 +624,10 @@ NETDATA_COMMON_LIBS = \

624 $(OPTIONAL_EBPF_LIBS) \

625 $(NULL)

626

627 +if LINK_STATIC_JSONC

628 + NETDATA_COMMON_LIBS += externaldeps/jsonc/libjson-c.a

629 +endif

630 +

631 NETDATACLI_FILES = \

632 daemon/commands.h \

633 $(LIBNETDATA_FILES) \

aclk/aclk_lws_wss_client.c

+32 -14

@@ -152,7 +152,6 @@ static void aclk_lws_wss_log_divert(int level, const char *line)

152 static int aclk_lws_wss_client_init( char *target_hostname, int target_port)

153 {

154 static int lws_logging_initialized = 0;

155 - struct lws_context_creation_info info;

155

156 if (unlikely(!lws_logging_initialized)) {

157 lws_set_log_level(LLL_ERR | LLL_WARN, aclk_lws_wss_log_divert);

@@ -167,14 +166,6 @@ static int aclk_lws_wss_client_init( char *target_hostname, int target_port)

166 engine_instance->host = target_hostname;

167 engine_instance->port = target_port;

168

170 - memset(&info, 0, sizeof(struct lws_context_creation_info));

171 - info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT;

172 - info.port = CONTEXT_PORT_NO_LISTEN;

173 - info.protocols = protocols;

174 -

175 - engine_instance->lws_context = lws_create_context(&info);

176 - if (!engine_instance->lws_context)

177 - goto failure_cleanup_2;

169

170 aclk_lws_mutex_init(&engine_instance->write_buf_mutex);

171 aclk_lws_mutex_init(&engine_instance->read_buf_mutex);

@@ -186,18 +177,27 @@ static int aclk_lws_wss_client_init( char *target_hostname, int target_port)

177 return 0;

178

179 failure_cleanup:

189 - lws_context_destroy(engine_instance->lws_context);

190 -failure_cleanup_2:

180 freez(engine_instance);

181 return 1;

182 }

183

195 -void aclk_lws_wss_client_destroy()

184 +void aclk_lws_wss_destroy_context()

185 {

197 - if (engine_instance == NULL)

186 + if (!engine_instance)

187 + return;

188 + if (!engine_instance->lws_context)

189 return;

190 lws_context_destroy(engine_instance->lws_context);

191 engine_instance->lws_context = NULL;

192 +}

193 +

194 +

195 +void aclk_lws_wss_client_destroy()

196 +{

197 + if (engine_instance == NULL)

198 + return;

199 +

200 + aclk_lws_wss_destroy_context();

201 engine_instance->lws_wsi = NULL;

202

203 aclk_lws_wss_clear_io_buffers(engine_instance);

@@ -267,7 +267,25 @@ int aclk_lws_wss_connect(char *host, int port)

267 int n;

268

269 if (!engine_instance) {

270 - return aclk_lws_wss_client_init(host, port);

270 + if (aclk_lws_wss_client_init(host, port))

271 + return 1; // Propagate failure

272 + }

273 +

274 + if (!engine_instance->lws_context)

275 + {

276 + // First time through (on this connection), create the context

277 + struct lws_context_creation_info info;

278 + memset(&info, 0, sizeof(struct lws_context_creation_info));

279 + info.options = LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT;

280 + info.port = CONTEXT_PORT_NO_LISTEN;

281 + info.protocols = protocols;

282 + engine_instance->lws_context = lws_create_context(&info);

283 + if (!engine_instance->lws_context)

284 + {

285 + error("Failed to create lws_context, ACLK will not function");

286 + return 1;

287 + }

288 + return 0;

289 // PROTOCOL_INIT callback will call again.

290 }

291

aclk/aclk_lws_wss_client.h

@@ -70,6 +70,7 @@ struct aclk_lws_wss_engine_instance {

70 };

72 void aclk_lws_wss_client_destroy();

73 +void aclk_lws_wss_destroy_context();

75 int aclk_lws_wss_connect(char *host, int port);

aclk/agent_cloud_link.c

+61 -32

@@ -23,6 +23,7 @@ static char *aclk_password = NULL;

23 static char *global_base_topic = NULL;

24 static int aclk_connecting = 0;

25 int aclk_connected = 0; // Exposed in the web-api

26 +int aclk_force_reconnect = 0; // Indication from lower layers

27 usec_t aclk_session_us = 0; // Used by the mqtt layer

28 time_t aclk_session_sec = 0; // Used by the mqtt layer

@@ -47,7 +48,7 @@ pthread_mutex_t query_lock_wait = PTHREAD_MUTEX_INITIALIZER;

48 #define QUERY_THREAD_WAKEUP pthread_cond_signal(&query_cond_wait)

50 void lws_wss_check_queues(size_t *write_len, size_t *write_len_bytes, size_t *read_len);

50 -

51 +void aclk_lws_wss_destroy_context();

52 /*

53 * Maintain a list of collectors and chart count

54 * If all the charts of a collector are deleted

@@ -149,7 +150,7 @@ static RSA *aclk_private_key = NULL;

150 static int create_private_key()

151 {

152 char filename[FILENAME_MAX + 1];

152 - snprintfz(filename, FILENAME_MAX, "%s/claim.d/private.pem", netdata_configured_user_config_dir);

153 + snprintfz(filename, FILENAME_MAX, "%s/cloud.d/private.pem", netdata_configured_varlib_dir);

154

155 long bytes_read;

156 char *private_key = read_by_filename(filename, &bytes_read);

@@ -1336,59 +1337,84 @@ void *aclk_main(void *ptr)

1337 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;

1338 struct netdata_static_thread *query_thread;

1339

1339 - if (!netdata_cloud_setting) {

1340 - info("Killing ACLK thread -> cloud functionality has been disabled");

1341 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

1342 - return NULL;

1343 - }

1340 + // This thread is unusual in that it cannot be cancelled by cancel_main_threads()

1341 + // as it must notify the far end that it shutdown gracefully and avoid the LWT.

1342 + netdata_thread_disable_cancelability();

1343 +

1344 +#if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK)

1345 + info("Killing ACLK thread -> cloud functionality has been disabled");

1346 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

1347 + return NULL;

1348 +#endif

1349

1350 info("Waiting for netdata to be ready");

1351 while (!netdata_ready) {

1352 sleep_usec(USEC_PER_MS * 300);

1353 }

1354

1355 + info("Waiting for Cloud to be enabled");

1356 + while (!netdata_cloud_setting) {

1357 + sleep_usec(USEC_PER_SEC * 1);

1358 + if (netdata_exit) {

1359 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

1360 + return NULL;

1361 + }

1362 + }

1363 +

1364 last_init_sequence = now_realtime_sec();

1365 query_thread = NULL;

1366

1367 char *aclk_hostname = NULL; // Initializers are over-written but prevent gcc complaining about clobbering.

1368 char *aclk_port = NULL;

1369 uint32_t port_num = 0;

1356 - char *cloud_base_url = config_get(CONFIG_SECTION_CLOUD, "cloud base url", DEFAULT_CLOUD_BASE_URL);

1357 - if (aclk_decode_base_url(cloud_base_url, &aclk_hostname, &aclk_port)) {

1358 - error("Configuration error - cannot use agent cloud link");

1359 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

1360 - return NULL;

1361 - }

1362 - port_num = atoi(aclk_port); // SSL library uses the string, MQTT uses the numeric value

1363 -

1370 info("Waiting for netdata to be claimed");

1371 while(1) {

1372 while (likely(!is_agent_claimed())) {

1367 - sleep_usec(USEC_PER_SEC * 5);

1373 + sleep_usec(USEC_PER_SEC * 1);

1374 if (netdata_exit)

1375 goto exited;

1376 }

1371 - if (!create_private_key() && !_mqtt_lib_init())

1372 - break;

1373 -

1374 - if (netdata_exit)

1377 + // The NULL return means the value was never initialised, but this value has been initialized in post_conf_load.

1378 + // We trap the impossible NULL here to keep the linter happy without using a fatal() in the code.

1379 + char *cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL);

1380 + if (cloud_base_url == NULL) {

1381 + error("Do not move the cloud base url out of post_conf_load!!");

1382 goto exited;

1383 + }

1384 + if (aclk_decode_base_url(cloud_base_url, &aclk_hostname, &aclk_port)) {

1385 + error("Agent is claimed but the configuration is invalid, please fix");

1386 + }

1387 + else

1388 + {

1389 + port_num = atoi(aclk_port); // SSL library uses the string, MQTT uses the numeric value

1390 + if (!create_private_key() && !_mqtt_lib_init())

1391 + break;

1392 + }

1393

1377 - sleep_usec(USEC_PER_SEC * 60);

1394 + for (int i=0; i<60; i++) {

1395 + if (netdata_exit)

1396 + goto exited;

1397 +

1398 + sleep_usec(USEC_PER_SEC * 1);

1399 + }

1400 }

1401 +

1402 create_publish_base_topic();

1403

1404 usec_t reconnect_expiry = 0; // In usecs

1405

1383 - netdata_thread_disable_cancelability();

1384 -

1406 while (!netdata_exit) {

1407 static int first_init = 0;

1408 size_t write_q, write_q_bytes, read_q;

1409 lws_wss_check_queues(&write_q, &write_q_bytes, &read_q);

1410 +

1411 + if (aclk_force_reconnect) {

1412 + aclk_lws_wss_destroy_context();

1413 + aclk_force_reconnect = 0;

1414 + }

1415 //info("loop state first_init_%d connected=%d connecting=%d wq=%zu (%zu-bytes) rq=%zu",

1416 // first_init, aclk_connected, aclk_connecting, write_q, write_q_bytes, read_q);

1391 - if (unlikely(!netdata_exit && !aclk_connected)) {

1417 + if (unlikely(!netdata_exit && !aclk_connected && !aclk_force_reconnect)) {

1418 if (unlikely(!first_init)) {

1419 aclk_try_to_connect(aclk_hostname, aclk_port, port_num);

1420 first_init = 1;

@@ -1414,7 +1440,7 @@ void *aclk_main(void *ptr)

1440 }

1441

1442 _link_event_loop();

1417 - if (unlikely(!aclk_connected))

1443 + if (unlikely(!aclk_connected || aclk_force_reconnect))

1444 continue;

1445 /*static int stress_counter = 0;

1446 if (write_q_bytes==0 && stress_counter ++ >5)

@@ -1550,6 +1576,7 @@ void aclk_disconnect()

1576 waiting_init = 1;

1577 aclk_connected = 0;

1578 aclk_connecting = 0;

1579 + aclk_force_reconnect = 1;

1580 }

1581

1582 void aclk_shutdown()

@@ -1598,6 +1625,7 @@ inline void aclk_create_header(BUFFER *dest, char *type, char *msg_id, time_t ts

1625 * alarm_log

1626 * active alarms

1627 */

1628 +void health_active_log_alarms_2json(RRDHOST *host, BUFFER *wb);

1629 void aclk_send_alarm_metadata()

1630 {

1631 BUFFER *local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);

@@ -1618,17 +1646,18 @@ void aclk_send_alarm_metadata()

1646 aclk_create_header(local_buffer, "connect_alarms", msg_id, aclk_session_sec, aclk_session_us);

1647 buffer_strcat(local_buffer, ",\n\t\"payload\": ");

1648

1649 +

1650 buffer_sprintf(local_buffer, "{\n\t \"configured-alarms\" : ");

1651 health_alarms2json(localhost, local_buffer, 1);

1652 debug(D_ACLK, "Metadata %s with configured alarms has %zu bytes", msg_id, local_buffer->len);

1653 + // buffer_sprintf(local_buffer, ",\n\t \"alarm-log\" : ");

1654 + // health_alarm_log2json(localhost, local_buffer, 0);

1655 + // debug(D_ACLK, "Metadata %s with alarm_log has %zu bytes", msg_id, local_buffer->len);

1656 + buffer_sprintf(local_buffer, ",\n\t \"alarms-active\" : ");

1657 + health_active_log_alarms_2json(localhost, local_buffer);

1658 + //debug(D_ACLK, "Metadata message %s", local_buffer->buffer);

1659

1625 - buffer_sprintf(local_buffer, ",\n\t \"alarm-log\" : ");

1626 - health_alarm_log2json(localhost, local_buffer, 0);

1627 - debug(D_ACLK, "Metadata %s with alarm_log has %zu bytes", msg_id, local_buffer->len);

1660

1629 - buffer_sprintf(local_buffer, ",\n\t \"alarms-active\" : ");

1630 - health_alarms_values2json(localhost, local_buffer, 0);

1631 - debug(D_ACLK, "Metadata %s with alarms_active has %zu bytes", msg_id, local_buffer->len);

1661

1662 buffer_sprintf(local_buffer, "\n}\n}");

1663 aclk_send_message(ACLK_ALARMS_TOPIC, local_buffer->buffer, msg_id);

@@ -1657,7 +1686,7 @@ int aclk_send_info_metadata()

1686 // a fake on_connect message then use the real timestamp to indicate it is within the existing

1687 // session.

1688 if (aclk_metadata_submitted == ACLK_METADATA_SENT)

1660 - aclk_create_header(local_buffer, "connect", msg_id, 0, 0);

1689 + aclk_create_header(local_buffer, "update", msg_id, 0, 0);

1690 else

1691 aclk_create_header(local_buffer, "connect", msg_id, aclk_session_sec, aclk_session_us);

1692 buffer_strcat(local_buffer, ",\n\t\"payload\": ");

aclk/agent_cloud_link.h

+1 -1

@@ -25,7 +25,7 @@

25 #define ACLK_MAX_TOPIC 255

27 #define ACLK_RECONNECT_DELAY 1 // reconnect delay -- with backoff stragegy fow now

28 -#define ACLK_STABLE_TIMEOUT 10 // Minimum delay to mark AGENT as stable

28 +#define ACLK_STABLE_TIMEOUT 3 // Minimum delay to mark AGENT as stable

29 #define ACLK_DEFAULT_PORT 9002

30 #define ACLK_DEFAULT_HOST "localhost"

aclk/mqtt.c

+3 -2

@@ -29,7 +29,7 @@ void publish_callback(struct mosquitto *mosq, void *obj, int rc)

29 UNUSED(mosq);

30 UNUSED(obj);

31 UNUSED(rc);

32 -

32 + info("Publish_callback: mid=%d", rc);

33 // TODO: link this with a msg_id so it can be traced

34 return;

35 }

@@ -219,7 +219,8 @@ void aclk_lws_connection_data_received()

219

220 void aclk_lws_connection_closed()

221 {

222 - aclk_disconnect(NULL);

222 + aclk_disconnect();

223 +

224 }

225

226

build/subst.inc

@@ -9,6 +9,8 @@

9 -e 's#[@]registrydir_POST@#$(registrydir)#g' \

10 -e 's#[@]varlibdir_POST@#$(varlibdir)#g' \

11 -e 's#[@]webdir_POST@#$(webdir)#g' \

12 + -e 's#[@]can_enable_aclk_POST@#$(can_enable_aclk)#g' \

13 + -e 's#[@]enable_cloud_POST@#$(enable_cloud)#g' \

14 $< > $@.tmp; then \

15 mv "$@.tmp" "$@"; \

16 else \

build_external/projects/aclk-testing/agent-compose.yml

+1 -1

@@ -9,7 +9,7 @@ services:

9 - VERSION=current

10 image: arch_current_dev:latest

11 command: >

12 - sh -c "echo -n 00000000-0000-0000-0000-000000000000 >/etc/netdata/claim.d/claimed_id &&

12 + sh -c "echo -n 00000000-0000-0000-0000-000000000000 >/var/lib/netdata/cloud.d/claimed_id &&

13 echo '[agent_cloud_link]' >>/etc/netdata/netdata.conf &&

14 echo ' agent cloud link hostname = vernemq' >>/etc/netdata/netdata.conf &&

15 echo ' agent cloud link port = 9002' >>/etc/netdata/netdata.conf &&

build_external/projects/aclk-testing/agent-valgrind-compose.yml

+1 -1

@@ -9,7 +9,7 @@ services:

9 - VERSION=extras

10 image: arch_extras_dev:latest

11 command: >

12 - sh -c "echo -n 00000000-0000-0000-0000-000000000000 >/etc/netdata/claim.d/claimed_id &&

12 + sh -c "echo -n 00000000-0000-0000-0000-000000000000 >/var/lib/netdata/cloud.d/claimed_id &&

13 echo '[agent_cloud_link]' >>/etc/netdata/netdata.conf &&

14 echo ' agent cloud link hostname = vernemq' >>/etc/netdata/netdata.conf &&

15 echo ' agent cloud link port = 9002' >>/etc/netdata/netdata.conf &&

claim/README.md

+15 -15

@@ -96,7 +96,7 @@ docker run -d --name=netdata \

96 --cap-add SYS_PTRACE \

97 --security-opt apparmor=unconfined \

98 netdata/netdata \

99 - /usr/sbin/netdata -D -W set global "netdata cloud" enable -W set cloud "cloud base url" "https://app.netdata.cloud" -W "claim -token=TOKEN -rooms=ROOM1,ROOM2 -url=https://app.netdata.cloud"

99 + /usr/sbin/netdata -D -W set cloud global enabled true -W set cloud global "cloud base url" "https://app.netdata.cloud" -W "claim -token=TOKEN -rooms=ROOM1,ROOM2 -url=https://app.netdata.cloud"

100 ```

101

102 The container runs in detached mode, so you won't see any output. If the node does not appear in your Space, you can run

@@ -167,11 +167,11 @@ Use these keys and the information below to troubleshoot the ACLK.

167

168 If `cloud-enabled` is `false`, you probably ran the installer with `--disable-cloud` option.

169

170 -Additionally, check that the `netdata cloud` setting in `netdata.conf` is set to `enable`:

170 +Additionally, check that the `enabled` setting in `var/lib/netdata/cloud.d/cloud.conf` is set to `true`:

171

172 ```ini

173 -[general]

174 - netadata cloud = enable

173 +[global]

174 + enabled = true

175 ```

176

177 To fix this issue, reinstall Netdata using your [preferred method](/packaging/installer/README.md) and do not add the

@@ -234,23 +234,23 @@ with details about your system and relevant output from `error.log`.

234

235 ### Unclaim (remove) an Agent from Netdata Cloud

236

237 -The best method to remove an Agent from Netdata Cloud is to unclaim it by deleting the `claim.d/` directory in your

238 -Netdata configuration directory.

237 +The best method to remove an Agent from Netdata Cloud is to unclaim it by deleting the `cloud.d/` directory in your

238 +Netdata library directory.

239

240 ```bash

241 -cd /etc/netdata # Replace with your Netdata configuration directory, if not /etc/netdata/

242 -rm -rf claim.d/

241 +cd /var/lib/netdata # Replace with your Netdata library directory, if not /var/lib/netdata/

242 +rm -rf cloud.d/

243 ```

244

245 > You may need to use `sudo` or another method of elevating your privileges.

246

247 -Once you delete the `claim.d/` directory, the ACLK will not connect to Cloud the next time the Agent starts, and Cloud

247 +Once you delete the `cloud.d/` directory, the ACLK will not connect to Cloud the next time the Agent starts, and Cloud

248 will then remove it from the interface.

249

250 ## Claiming reference

251

252 In the sections below, you can find reference material for the claiming script, claiming via the Agent's command line

253 -tool, and details about the files found in `claim.d`.

253 +tool, and details about the files found in `cloud.d`.

254

255 ### Claiming script

256

@@ -263,7 +263,7 @@ and passing the following arguments:

263 -rooms=ROOM1,ROOM2,...

264 where ROOMX is the War Room this node should be added to. This list is optional.

265 -url=URL_BASE

266 - where URL_BASE is the Netdata Cloud endpoint base URL. By default, this is https://netdata.cloud.

266 + where URL_BASE is the Netdata Cloud endpoint base URL. By default, this is https://app.netdata.cloud.

267 -id=AGENT_ID

268 where AGENT_ID is the unique identifier of the Agent. This is the Agent's MACHINE_GUID by default.

269 -hostname=HOSTNAME

@@ -306,14 +306,14 @@ If need be, the user can override the Agent's defaults by providing additional a

306

307 ### Claiming directory

308

309 -Netdata stores the agent claiming-related state in the user configuration directory under `claim.d`, e.g. in

310 -`/etc/netdata/claim.d`. The user can put files in this directory to provide defaults to the `-token` and `-rooms`

309 +Netdata stores the agent claiming-related state in the Netdata library directory under `cloud.d`, e.g. in

310 +`/var/lib/netdata/cloud.d`. The user can put files in this directory to provide defaults to the `-token` and `-rooms`

311 arguments. These files should be owned **by the `netdata` user**.

312

313 -The `claim.d/token` file should contain the claiming-token and the `claim.d/rooms` file should contain the list of

313 +The `cloud.d/token` file should contain the claiming-token and the `cloud.d/rooms` file should contain the list of

314 war-rooms.

315

316 -The user can also put the Cloud endpoint's full certificate chain in `claim.d/cloud_fullchain.pem` so that the Agent

316 +The user can also put the Cloud endpoint's full certificate chain in `cloud.d/cloud_fullchain.pem` so that the Agent

317 can trust the endpoint if necessary.

318

319 [![analytics](https://www.google-analytics.com/collect?v=1&aip=1&t=pageview&_s=1&ds=github&dr=https%3A%2F%2Fgithub.com%2Fnetdata%2Fnetdata&dl=https%3A%2F%2Fmy-netdata.io%2Fgithub%2Fclaim%2FREADME&_u=MAC~&cid=5792dfd7-8dc4-476b-af31-da2fdb9f93d2&tid=UA-64295674-3)](<>)

claim/claim.c

+52 -14

@@ -12,17 +12,19 @@ static char *claiming_errors[] = {

12 "Problems with claiming working directory", // 2

13 "Missing dependencies", // 3

14 "Failure to connect to endpoint", // 4

15 - "Unknown HTTP error message", // 5

16 - "invalid node id", // 6

17 - "invalid node name", // 7

18 - "invalid room id", // 8

19 - "invalid public key", // 9

20 - "token expired/token not found/invalid token", // 10

21 - "already claimed", // 11

22 - "processing claiming", // 12

23 - "Internal Server Error", // 13

24 - "Gateway Timeout", // 14

25 - "Service Unavailable" // 15

15 + "The CLI didn't work", // 5

16 + "Wrong user", // 6

17 + "Unknown HTTP error message", // 7

18 + "invalid node id", // 8

19 + "invalid node name", // 9

20 + "invalid room id", // 10

21 + "invalid public key", // 11

22 + "token expired/token not found/invalid token", // 12

23 + "already claimed", // 13

24 + "processing claiming", // 14

25 + "Internal Server Error", // 15

26 + "Gateway Timeout", // 16

27 + "Service Unavailable" // 17

28 };

30 static char *claimed_id = NULL;

@@ -37,7 +39,7 @@ char *is_agent_claimed(void)

40 extern struct registry registry;

40 -/* rrd_init() must have been called before this function */

42 +/* rrd_init() and post_conf_load() must have been called before this function */

43 void claim_agent(char *claiming_arguments)

44 {

45 if (!netdata_cloud_setting) {

@@ -51,7 +53,10 @@ void claim_agent(char *claiming_arguments)

53 char command_buffer[CLAIMING_COMMAND_LENGTH + 1];

54 FILE *fp;

54 - char *cloud_base_url = config_get(CONFIG_SECTION_CLOUD, "cloud base url", DEFAULT_CLOUD_BASE_URL);

56 + // This is guaranteed to be set early in main via post_conf_load()

57 + char *cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL);

58 + if (cloud_base_url == NULL)

59 + fatal("Do not move the cloud base url out of post_conf_load!!");

60 const char *proxy_str;

61 ACLK_PROXY_TYPE proxy_type;

62 char proxy_flag[CLAIMING_PROXY_LENGTH] = "-noproxy";

@@ -111,8 +116,11 @@ void load_claiming_state(void)

116 claimed_id = NULL;

117 }

118

119 + // Propagate into aclk and registry. Be kind of atomic...

120 + appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", DEFAULT_CLOUD_BASE_URL);

121 +

122 char filename[FILENAME_MAX + 1];

115 - snprintfz(filename, FILENAME_MAX, "%s/claim.d/claimed_id", netdata_configured_user_config_dir);

123 + snprintfz(filename, FILENAME_MAX, "%s/cloud.d/claimed_id", netdata_configured_varlib_dir);

124

125 long bytes_read;

126 claimed_id = read_by_filename(filename, &bytes_read);

@@ -122,4 +130,34 @@ void load_claiming_state(void)

130 }

131

132 info("File '%s' was found. Setting state to AGENT_CLAIMED.", filename);

133 +

134 + // --------------------------------------------------------------------

135 + // Check if the cloud is enabled

136 +#if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK )

137 + netdata_cloud_setting = 0;

138 +#else

139 + netdata_cloud_setting = appconfig_get_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", 1);

140 +#endif

141 +}

142 +

143 +struct config cloud_config = { .first_section = NULL,

144 + .last_section = NULL,

145 + .mutex = NETDATA_MUTEX_INITIALIZER,

146 + .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },

147 + .rwlock = AVL_LOCK_INITIALIZER } };

148 +

149 +void load_cloud_conf(int silent)

150 +{

151 + char *filename;

152 + errno = 0;

153 +

154 + int ret = 0;

155 +

156 + filename = strdupz_path_subpath(netdata_configured_varlib_dir, "cloud.d/cloud.conf");

157 +

158 + ret = appconfig_load(&cloud_config, filename, 1, NULL);

159 + if(!ret && !silent) {

160 + info("CONFIG: cannot load cloud config '%s'. Running with internal defaults.", filename);

161 + }

162 + freez(filename);

163 }

claim/claim.h

@@ -6,9 +6,11 @@

6 #include "../daemon/common.h"

8 extern char *claiming_pending_arguments;

9 +extern struct config cloud_config;

11 void claim_agent(char *claiming_arguments);

12 char *is_agent_claimed(void);

13 void load_claiming_state(void);

14 +void load_cloud_conf(int silent);

16 #endif //NETDATA_CLAIM_H

claim/netdata-claim.sh.in

+130 -70

@@ -9,74 +9,92 @@

9 # Exit code: 2 - Problems with claiming working directory

10 # Exit code: 3 - Missing dependencies

11 # Exit code: 4 - Failure to connect to endpoint

12 -# Exit code: 5 - Unknown HTTP error message

13 -# Exit code: 6 - The CLI didn't work

14 -# Exit code: 7 - Wrong user

12 +# Exit code: 5 - The CLI didn't work

13 +# Exit code: 6 - Wrong user

14 +# Exit code: 7 - Unknown HTTP error message

15 #

16 # OK: Agent claimed successfully

17 # HTTP Status code: 204

18 # Exit code: 0

19 #

20 +# Unknown HTTP error message

21 +# HTTP Status code: 422

22 +# Exit code: 7

23 +ERROR_KEYS[7]="None"

24 +ERROR_MESSAGES[7]="Unknown HTTP error message"

25 +

26 # Error: The agent id is invalid; it does not fulfill the constraints

27 # HTTP Status code: 422

22 -# Error key: "ErrInvalidNodeID"

23 -# Error message: "invalid node id"

24 -# Exit code: 6

28 +# Exit code: 8

29 +ERROR_KEYS[8]="ErrInvalidNodeID"

30 +ERROR_MESSAGES[8]="invalid node id"

32 # Error: The agent hostname is invalid; it does not fulfill the constraints

33 # HTTP Status code: 422

28 -# Error key: "ErrInvalidNodeName"

29 -# Error message: "invalid node name"

30 -# Exit code: 7

31 -#

34 +# Exit code: 9

35 +ERROR_KEYS[9]="ErrInvalidNodeName"

36 +ERROR_MESSAGES[9]="invalid node name"

37 +

38 # Error: At least one of the given rooms ids is invalid; it does not fulfill the constraints

39 # HTTP Status code: 422

34 -# Error key: "ErrInvalidRoomID"

35 -# Error message: "invalid room id"

36 -# Exit code: 8

37 -#

40 +# Exit code: 10

41 +ERROR_KEYS[10]="ErrInvalidRoomID"

42 +ERROR_MESSAGES[10]="invalid room id"

43 +

44 # Error: Invalid public key; the public key is empty or not present

45 # HTTP Status code: 422

40 -# Error key: "ErrInvalidPublicKey"

41 -# Error message: "invalid public key"

42 -# Exit code: 9

46 +# Exit code: 11

47 +ERROR_KEYS[11]="ErrInvalidPublicKey"

48 +ERROR_MESSAGES[11]="invalid public key"

49 #

50 # Error: Expired, missing or invalid token

51 # HTTP Status code: 403

46 -# Error key: "ErrForbidden"

47 -# Error message: "token expired" | "token not found" | "invalid token"

48 -# Exit code: 10

49 -#

52 +# Exit code: 12

53 +ERROR_KEYS[12]="ErrForbidden"

54 +ERROR_MESSAGES[12]="token expired/token not found/invalid token"

55 +

56 # Error: Duplicate agent id; an agent with the same id is already registered in the cloud

57 # HTTP Status code: 409

52 -# Error key: "ErrAlreadyClaimed"

53 -# Error message: "already claimed"

54 -# Exit code: 11

55 -#

58 +# Exit code: 13

59 +ERROR_KEYS[13]="ErrAlreadyClaimed"

60 +ERROR_MESSAGES[13]="already claimed"

61 +

62 # Error: The node claiming process is still in progress.

63 # HTTP Status code: 102

58 -# Error key: "ErrProcessingClaim"

59 -# Error message: "processing claiming"

60 -# Exit code: 12

61 -#

64 +# Exit code: 14

65 +ERROR_KEYS[14]="ErrProcessingClaim"

66 +ERROR_MESSAGES[14]="processing claiming"

67 +

68 # Error: Internal server error. Any other unexpected error (DB problems, etc.)

69 # HTTP Status code: 500

64 -# Error key: "ErrInternalServerError"

65 -# Error message: "Internal Server Error"

66 -# Exit code: 13

67 -#

70 +# Exit code: 15

71 +ERROR_KEYS[15]="ErrInternalServerError"

72 +ERROR_MESSAGES[15]="Internal Server Error"

73 +

74 # Error: There was a timout processing the claim.

75 # HTTP Status code: 504

70 -# Error key: "ErrGatewayTimeout"

71 -# Error message: "Gateway Timeout"

72 -# Exit code: 14

73 -#

76 +# Exit code: 16

77 +ERROR_KEYS[16]="ErrGatewayTimeout"

78 +ERROR_MESSAGES[16]="Gateway Timeout"

79 +

80 # Error: The service cannot handle the claiming request at this time.

81 # HTTP Status code: 503

76 -# Error key: "ErrServiceUnavailable"

77 -# Error message: "Service Unavailable"

78 -# Exit code: 15

82 +# Exit code: 17

83 +ERROR_KEYS[17]="ErrServiceUnavailable"

84 +ERROR_MESSAGES[17]="Service Unavailable"

86 +get_config_value() {

87 + conf_file="${1}"

88 + section="${2}"

89 + key_name="${3}"

90 + config_result=$(@sbindir_POST@/netdatacli 2>/dev/null read-config "$conf_file|$section|$key_name"; exit $?)

91 + # shellcheck disable=SC2181

92 + if [ "$?" != "0" ]; then

93 + echo >&2 "cli failed, assume netdata is not running and query the on-disk config"

94 + config_result=$(@sbindir_POST@/netdata 2>/dev/null -W get2 "$conf_file" "$section" "$key_name" unknown_default)

95 + fi

96 + echo "$config_result"

97 +}

98 if command -v curl >/dev/null 2>&1 ; then

99 URLTOOL="curl"

100 elif command -v wget >/dev/null 2>&1 ; then

@@ -90,15 +108,26 @@ if ! command -v openssl >/dev/null 2>&1 ; then

108 exit 3

109 fi

110

111 +# shellcheck disable=SC2050

112 +if [ "@enable_cloud_POST@" = "no" ]; then

113 + echo >&2 "This agent was built with --disable-cloud and cannot be claimed"

114 + exit 3

115 +fi

116 +# shellcheck disable=SC2050

117 +if [ "@can_enable_aclk_POST@" != "yes" ]; then

118 + echo >&2 "This agent was built without the dependencies for Cloud and cannot be claimed"

119 + exit 3

120 +fi

121

122 # -----------------------------------------------------------------------------

123 # defaults to allow running this script by hand

124

97 -[ -z "${NETDATA_USER_CONFIG_DIR}" ] && NETDATA_USER_CONFIG_DIR="@configdir_POST@"

125 +[ -z "${NETDATA_VARLIB_DIR}" ] && NETDATA_VARLIB_DIR="@varlibdir_POST@"

126 MACHINE_GUID_FILE="@registrydir_POST@/netdata.public.unique.id"

99 -CLAIMING_DIR="${NETDATA_USER_CONFIG_DIR}/claim.d"

127 +CLAIMING_DIR="${NETDATA_VARLIB_DIR}/cloud.d"

128 TOKEN="unknown"

101 -URL_BASE="https://netdata.cloud"

129 +URL_BASE=$(get_config_value cloud global "cloud base url")

130 +[ -z "$URL_BASE" ] && URL_BASE="https://app.netdata.cloud" # Cover post-install with --dont-start

131 ID="unknown"

132 ROOMS=""

133 [ -z "$HOSTNAME" ] && HOSTNAME=$(hostname)

@@ -106,14 +135,9 @@ CLOUD_CERTIFICATE_FILE="${CLAIMING_DIR}/cloud_fullchain.pem"

135 VERBOSE=0

136 INSECURE=0

137 RELOAD=1

109 -NETDATA_USER=netdata

138 +NETDATA_USER=$(get_config_value netdata global "run as user")

139 [ -z "$EUID" ] && EUID="$(id -u)"

140

112 -CONF_USER=$(grep '^[ #]*run as user[ ]*=' "${NETDATA_USER_CONFIG_DIR}/netdata.conf" 2>/dev/null)

113 -if [ -n "$CONF_USER" ]; then

114 - NETDATA_USER=$(echo "$CONF_USER" | sed 's/^[^=]*=[ \t]*//' | sed 's/[ \t]*$//')

115 -fi

116 -

141

142 # get the MACHINE_GUID by default

143 if [ -r "${MACHINE_GUID_FILE}" ]; then

@@ -152,7 +176,7 @@ done

176

177 if [ "$EUID" != "0" ] && [ "$(whoami)" != "$NETDATA_USER" ]; then

178 echo >&2 "This script must be run by the $NETDATA_USER user account"

155 - exit 7

179 + exit 6

180 fi

181

182 # if curl not installed give warning SOCKS can't be used

@@ -279,37 +303,73 @@ if [ "${VERBOSE}" == 1 ] ; then

303 cat "${CLAIMING_DIR}/tmpout.txt"

304 fi

305

282 -HTTP_STATUS_CODE=$(grep "HTTP" "${CLAIMING_DIR}/tmpout.txt" | awk -F " " '{print $2}')

306 +ERROR_KEY=$(grep "\"errorMsgKey\":" "${CLAIMING_DIR}/tmpout.txt" | awk -F "errorMsgKey\":\"" '{print $2}' | awk -F "\"" '{print $1}')

307 +case ${ERROR_KEY} in

308 + "ErrInvalidNodeID") EXIT_CODE=8 ;;

309 + "ErrInvalidNodeName") EXIT_CODE=9 ;;

310 + "ErrInvalidRoomID") EXIT_CODE=10 ;;

311 + "ErrInvalidPublicKey") EXIT_CODE=11 ;;

312 + "ErrForbidden") EXIT_CODE=12 ;;

313 + "ErrAlreadyClaimed") EXIT_CODE=13 ;;

314 + "ErrProcessingClaim") EXIT_CODE=14 ;;

315 + "ErrInternalServerError") EXIT_CODE=15 ;;

316 + "ErrGatewayTimeout") EXIT_CODE=16 ;;

317 + "ErrServiceUnavailable") EXIT_CODE=17 ;;

318 + *) EXIT_CODE=7 ;;

319 +esac

320

321 +HTTP_STATUS_CODE=$(grep "HTTP" "${CLAIMING_DIR}/tmpout.txt" | awk -F " " '{print $2}')

322 if [ "${HTTP_STATUS_CODE}" = "204" ] ; then

323 + EXIT_CODE=0

324 +fi

325 +

326 +if [ "${HTTP_STATUS_CODE}" = "204" ] || [ "${ERROR_KEY}" = "ErrAlreadyClaimed" ] ; then

327 rm -f "${CLAIMING_DIR}/tmpout.txt"

328 echo -n "${ID}" >"${CLAIMING_DIR}/claimed_id" || (echo >&2 "Claiming failed"; set -e; exit 2)

329 rm -f "${CLAIMING_DIR}/token" || (echo >&2 "Claiming failed"; set -e; exit 2)

330 +

331 + # Rewrite the cloud.conf on the disk

332 + cat > "$CLAIMING_DIR/cloud.conf" <<HERE_DOC

333 +[global]

334 + enabled = yes

335 + cloud base url = $URL_BASE

336 +HERE_DOC

337 if [ "$EUID" == "0" ]; then

338 chown -R "${NETDATA_USER}:${NETDATA_USER}" ${CLAIMING_DIR} || (echo >&2 "Claiming failed"; set -e; exit 2)

339 fi

340 if [ "${RELOAD}" == "0" ] ; then

292 - exit 0

341 + exit $EXIT_CODE

342 + fi

343 +

344 + if [ -z "${PROXY}" ]; then

345 + PROXYMSG=""

346 + else

347 + PROXYMSG="You have attempted to claim this node through a proxy - please update your the proxy setting in your netdata.conf to ${PROXY}. "

348 fi

294 - netdatacli reload-claiming-state && echo >&2 "Node was successfully claimed." && exit 0

295 - echo "The claim was successful but the agent could not be notified ($?)- it requires a restart to connect to the cloud"

296 - exit 6

349 + # Update cloud.conf in the agent memory

350 + @sbindir_POST@/netdatacli write-config 'cloud|global|enabled|yes' && \

351 + @sbindir_POST@/netdatacli write-config "cloud|global|cloud base url|$URL_BASE" && \

352 + @sbindir_POST@/netdatacli reload-claiming-state && \

353 + if [ "${HTTP_STATUS_CODE}" = "204" ] ; then

354 + echo >&2 "${PROXYMSG}Node was successfully claimed."

355 + else

356 + echo >&2 "The agent cloud base url is set to the url provided."

357 + echo >&2 "The cloud may have different credentials already registered for this agent ID and it cannot be reclaimed under different credentials for security reasons. If you are unable to connect use -id=\$(uuidgen) to overwrite this agent ID with a fresh value if the original credentials cannot be restored."

358 + echo >&2 "${PROXYMSG}Failed to claim node with the following error message:\"${ERROR_MESSAGES[$EXIT_CODE]}\""

359 + fi && exit $EXIT_CODE

360 +

361 + if [ "${ERROR_KEY}" = "ErrAlreadyClaimed" ] ; then

362 + echo >&2 "The cloud may have different credentials already registered for this agent ID and it cannot be reclaimed under different credentials for security reasons. If you are unable to connect use -id=\$(uuidgen) to overwrite this agent ID with a fresh value if the original credentials cannot be restored."

363 + echo >&2 "${PROXYMSG}Failed to claim node with the following error message:\"${ERROR_MESSAGES[$EXIT_CODE]}\""

364 + exit $EXIT_CODE

365 + fi

366 + echo >&2 "${PROXYMSG}The claim was successful but the agent could not be notified ($?)- it requires a restart to connect to the cloud."

367 + exit 5

368 fi

369

299 -ERROR_MESSAGE=$(grep "\"errorMsgKey\":" "${CLAIMING_DIR}/tmpout.txt" | awk -F "errorMsgKey\":\"" '{print $2}' | awk -F "\"" '{print $1}')

300 -case ${ERROR_MESSAGE} in

301 - "ErrInvalidNodeID") EXIT_CODE=6 ;;

302 - "ErrInvalidNodeName") EXIT_CODE=7 ;;

303 - "ErrInvalidRoomID") EXIT_CODE=8 ;;

304 - "ErrInvalidPublicKey") EXIT_CODE=9 ;;

305 - "ErrForbidden") EXIT_CODE=10 ;;

306 - "ErrAlreadyClaimed") EXIT_CODE=11 ;;

307 - "ErrProcessingClaim") EXIT_CODE=12 ;;

308 - "ErrInternalServerError") EXIT_CODE=13 ;;

309 - "ErrGatewayTimeout") EXIT_CODE=14 ;;

310 - "ErrServiceUnavailable") EXIT_CODE=15 ;;

311 - *) EXIT_CODE=5 ;;

312 -esac

313 -echo >&2 "Failed to claim node."

370 +echo >&2 "Failed to claim node with the following error message:\"${ERROR_MESSAGES[$EXIT_CODE]}\""

371 +if [ "${VERBOSE}" == 1 ]; then

372 + echo >&2 "Error key was:\"${ERROR_KEYS[$EXIT_CODE]}\""

373 +fi

374 rm -f "${CLAIMING_DIR}/tmpout.txt"

375 exit $EXIT_CODE

configure.ac

+45 -7

@@ -173,7 +173,6 @@ AC_ARG_ENABLE(

173 [ enable_cloud="detect" ]

174 )

175

176 -aclk_required="${enable_cloud}"

176 if test "${enable_cloud}" = "no"; then

177 AC_DEFINE([DISABLE_CLOUD], [1], [disable netdata cloud functionality])

178 fi

@@ -435,6 +434,35 @@ AM_CONDITIONAL([ENABLE_HTTPS], [test "${enable_https}" = "yes"])

434

435 # -----------------------------------------------------------------------------

436 # JSON-C

437 +

438 +if test "${enable_jsonc}" != "no" -a -z "${JSONC_LIBS}"; then

439 + # Try and detect manual static build presence (from netdata-installer.sh)

440 + AC_MSG_CHECKING([if statically built json-c is present])

441 + HAVE_libjson_c_a="no"

442 + if test -f "externaldeps/jsonc/libjson-c.a"; then

443 + LIBS_BKP="${LIBS}"

444 + LIBS="externaldeps/jsonc/libjson-c.a"

445 + AC_LINK_IFELSE([AC_LANG_SOURCE([[#include "externaldeps/jsonc/json-c/json.h"

446 + int main (int argc, char **argv) {

447 + struct json_object *jobj;

448 + char *str = "{ \"msg-type\": \"random\" }";

449 + jobj = json_tokener_parse(str);

450 + json_object_get_type(jobj);

451 + }]])],

452 + [HAVE_libjson_c_a="yes"],

453 + [HAVE_libjson_c_a="no"])

454 + LIBS="${LIBS_BKP}"

455 + fi

456 +

457 + if test "${HAVE_libjson_c_a}" = "yes"; then

458 + AC_DEFINE([LINK_STATIC_JSONC], [1], [static json-c should be used])

459 + JSONC_LIBS="static"

460 + OPTIONAL_JSONC_STATIC_CFLAGS="-I externaldeps/jsonc"

461 + fi

462 + AC_MSG_RESULT([${HAVE_libjson_c_a}])

463 +fi

464 +AM_CONDITIONAL([LINK_STATIC_JSONC], [test "${JSONC_LIBS}" = "static"])

465 +

466 test "${enable_jsonc}" = "yes" -a -z "${JSONC_LIBS}" && \

467 AC_MSG_ERROR([JSON-C required but not found. Try installing 'libjson-c-dev' or 'json-c'.])

468

@@ -577,7 +605,7 @@ if test "$enable_cloud" != "no"; then

605 fi

606 AC_MSG_RESULT([${HAVE_libwebsockets_a}])

607

580 - if test "${build_target}" = "linux" -a "${aclk_required}" != "no"; then

608 + if test "${build_target}" = "linux" -a "${enable_cloud}" != "no"; then

609 if test "${have_libcap}" = "yes" -a "${with_libcap}" = "no"; then

610 AC_MSG_ERROR([agent-cloud-link can't be built without libcap. Disable it by --disable-cloud or enable libcap])

611 fi

@@ -586,23 +614,31 @@ if test "$enable_cloud" != "no"; then

614 fi

615 fi

616

617 + # next 2 lines are just to have info for ACLK dependencies in common place

618 + AC_MSG_CHECKING([if json-c available for ACLK])

619 + AC_MSG_RESULT([${enable_jsonc}])

620 +

621 + test "${enable_cloud}" = "yes" -a "${enable_jsonc}" = "no" && \

622 + AC_MSG_ERROR([You have asked for ACLK to be built but no json-c available. ACLK requires json-c])

623 +

624 AC_MSG_CHECKING([if netdata agent-cloud-link can be enabled])

590 - if test "${HAVE_libmosquitto_a}" = "yes" -a "${HAVE_libwebsockets_a}" = "yes" -a -n "${SSL_LIBS}"; then

625 + if test "${HAVE_libmosquitto_a}" = "yes" -a "${HAVE_libwebsockets_a}" = "yes" -a -n "${SSL_LIBS}" -a "${enable_jsonc}" = "yes"; then

626 can_enable_aclk="yes"

627 else

628 can_enable_aclk="no"

629 fi

630 AC_MSG_RESULT([${can_enable_aclk}])

631

597 - test "${aclk_required}" = "yes" -a "${can_enable_aclk}" = "no" && \

632 + test "${enable_cloud}" = "yes" -a "${can_enable_aclk}" = "no" && \

633 AC_MSG_ERROR([User required agent-cloud-link but it can't be built!])

634

635 AC_MSG_CHECKING([if netdata agent-cloud-link should/will be enabled])

601 - if test "${aclk_required}" = "detect"; then

636 + if test "${enable_cloud}" = "detect"; then

637 enable_aclk=$can_enable_aclk

638 else

604 - enable_aclk=$aclk_required

639 + enable_aclk=$enable_cloud

640 fi

641 + AC_SUBST([can_enable_aclk])

642

643 if test "${enable_aclk}" = "yes"; then

644 AC_DEFINE([ENABLE_ACLK], [1], [netdata ACLK])

@@ -610,6 +646,7 @@ if test "$enable_cloud" != "no"; then

646

647 AC_MSG_RESULT([${enable_aclk}])

648 fi

649 +AC_SUBST([enable_cloud])

650 AM_CONDITIONAL([ENABLE_ACLK], [test "${enable_aclk}" = "yes"])

651

652 # -----------------------------------------------------------------------------

@@ -1216,7 +1253,8 @@ AC_SUBST([webdir])

1253

1254 CFLAGS="${CFLAGS} ${OPTIONAL_MATH_CFLAGS} ${OPTIONAL_NFACCT_CFLAGS} ${OPTIONAL_ZLIB_CFLAGS} ${OPTIONAL_UUID_CFLAGS} \

1255 ${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} \

1219 - ${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS} ${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS}"

1256 + ${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS} ${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS} \

1257 + ${OPTIONAL_JSONC_STATIC_CFLAGS}"

1258

1259 CXXFLAGS="${CFLAGS} ${CXX11FLAG}"

1260

daemon/commands.c

+85 -14

@@ -43,6 +43,8 @@ static cmd_status_t cmd_exit_execute(char *args, char **message);

43 static cmd_status_t cmd_fatal_execute(char *args, char **message);

44 static cmd_status_t cmd_reload_claiming_state_execute(char *args, char **message);

45 static cmd_status_t cmd_reload_labels_execute(char *args, char **message);

46 +static cmd_status_t cmd_read_config_execute(char *args, char **message);

47 +static cmd_status_t cmd_write_config_execute(char *args, char **message);

49 static command_info_t command_info_array[] = {

50 {"help", cmd_help_execute, CMD_TYPE_HIGH_PRIORITY}, // show help menu

@@ -53,6 +55,8 @@ static command_info_t command_info_array[] = {

55 {"fatal-agent", cmd_fatal_execute, CMD_TYPE_HIGH_PRIORITY}, // exit with fatal error

56 {"reload-claiming-state", cmd_reload_claiming_state_execute, CMD_TYPE_ORTHOGONAL}, // reload claiming state

57 {"reload-labels", cmd_reload_labels_execute, CMD_TYPE_ORTHOGONAL}, // reload the labels

58 + {"read-config", cmd_read_config_execute, CMD_TYPE_CONCURRENT},

59 + {"write-config", cmd_write_config_execute, CMD_TYPE_ORTHOGONAL}

60 };

62 /* Mutexes for commands of type CMD_TYPE_ORTHOGONAL */

@@ -185,23 +189,15 @@ static cmd_status_t cmd_reload_claiming_state_execute(char *args, char **message

189 {

190 (void)args;

191 (void)message;

188 -

189 -#ifdef DISABLE_CLOUD

190 - info("The claiming feature has been disabled");

191 - return CMD_STATUS_FAILURE;

192 -#endif

193 -#ifndef ENABLE_ACLK

194 - info("Cloud functionality is not enabled because of missing dependencies at build-time.");

192 +#if defined(DISABLE_CLOUD) || !defined(ENABLE_ACLK)

193 + info("The claiming feature has been explicitly disabled");

194 + *message = strdupz("This agent cannot be claimed, it was built without support for Cloud");

195 return CMD_STATUS_FAILURE;

196 #endif

197 - if (!netdata_cloud_setting) {

198 - error("Cannot reload claiming status -> cloud functionality has been disabled");

199 - return CMD_STATUS_FAILURE;

200 - }

201 -

197 error_log_limit_unlimited();

198 info("COMMAND: Reloading Agent Claiming configuration.");

199 load_claiming_state();

200 + registry_update_cloud_base_url();

201 error_log_limit_reset();

202 return CMD_STATUS_SUCCESS;

203 }

@@ -230,6 +226,76 @@ static cmd_status_t cmd_reload_labels_execute(char *args, char **message)

226 return CMD_STATUS_SUCCESS;

227 }

228

229 +static cmd_status_t cmd_read_config_execute(char *args, char **message)

230 +{

231 + size_t n = strlen(args);

232 + char *separator = strchr(args,'|');

233 + if (separator == NULL)

234 + return CMD_STATUS_FAILURE;

235 + char *separator2 = strchr(separator + 1,'|');

236 + if (separator2 == NULL)

237 + return CMD_STATUS_FAILURE;

238 +

239 + char *temp = callocz(n + 1, 1);

240 + strcpy(temp, args);

241 + size_t offset = separator - args;

242 + temp[offset] = 0;

243 + size_t offset2 = separator2 - args;

244 + temp[offset2] = 0;

245 +

246 + const char *conf_file = temp; /* "cloud" is cloud.conf, otherwise netdata.conf */

247 + struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config;

248 +

249 + char *value = appconfig_get(tmp_config, temp + offset + 1, temp + offset2 + 1, NULL);

250 + if (value == NULL)

251 + {

252 + error("Cannot execute read-config conf_file=%s section=%s / key=%s because no value set", conf_file,

253 + temp + offset + 1, temp + offset2 + 1);

254 + freez(temp);

255 + return CMD_STATUS_FAILURE;

256 + }

257 + else

258 + {

259 + (*message) = strdupz(value);

260 + freez(temp);

261 + return CMD_STATUS_SUCCESS;

262 + }

263 +

264 +}

265 +

266 +static cmd_status_t cmd_write_config_execute(char *args, char **message)

267 +{

268 + UNUSED(message);

269 + info("write-config %s", args);

270 + size_t n = strlen(args);

271 + char *separator = strchr(args,'|');

272 + if (separator == NULL)

273 + return CMD_STATUS_FAILURE;

274 + char *separator2 = strchr(separator + 1,'|');

275 + if (separator2 == NULL)

276 + return CMD_STATUS_FAILURE;

277 + char *separator3 = strchr(separator2 + 1,'|');

278 + if (separator3 == NULL)

279 + return CMD_STATUS_FAILURE;

280 + char *temp = callocz(n + 1, 1);

281 + strcpy(temp, args);

282 + size_t offset = separator - args;

283 + temp[offset] = 0;

284 + size_t offset2 = separator2 - args;

285 + temp[offset2] = 0;

286 + size_t offset3 = separator3 - args;

287 + temp[offset3] = 0;

288 +

289 + const char *conf_file = temp; /* "cloud" is cloud.conf, otherwise netdata.conf */

290 + struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config;

291 +

292 + appconfig_set(tmp_config, temp + offset + 1, temp + offset2 + 1, temp + offset3 + 1);

293 + info("write-config conf_file=%s section=%s key=%s value=%s",conf_file, temp + offset + 1, temp + offset2 + 1,

294 + temp + offset3 + 1);

295 + freez(temp);

296 + return CMD_STATUS_SUCCESS;

297 +}

298 +

299 static void cmd_lock_exclusive(unsigned index)

300 {

301 (void)index;

@@ -369,9 +435,11 @@ static void schedule_command(uv_work_t *req)

435 cmd_ctx->status = execute_command(cmd_ctx->idx, cmd_ctx->args, &cmd_ctx->message);

436 }

437

438 +/* This will alter the state of the command_info_array.cmd_str

439 +*/

440 static void parse_commands(struct command_context *cmd_ctx)

441 {

374 - char *message = NULL, *pos;

442 + char *message = NULL, *pos, *lstrip, *rstrip;

443 cmd_t i;

444 cmd_status_t status;

445

@@ -381,9 +449,12 @@ static void parse_commands(struct command_context *cmd_ctx)

449 for (pos = cmd_ctx->command_string ; isspace(*pos) && ('\0' != *pos) ; ++pos) {;}

450 for (i = 0 ; i < CMD_TOTAL_COMMANDS ; ++i) {

451 if (!strncmp(pos, command_info_array[i].cmd_str, strlen(command_info_array[i].cmd_str))) {

452 + for (lstrip=pos + strlen(command_info_array[i].cmd_str); isspace(*lstrip) && ('\0' != *lstrip); ++lstrip) {;}

453 + for (rstrip=lstrip+strlen(lstrip)-1; rstrip>lstrip && isspace(*rstrip); *(rstrip--) = 0 );

454 +

455 cmd_ctx->work.data = cmd_ctx;

456 cmd_ctx->idx = i;

386 - cmd_ctx->args = pos + strlen(command_info_array[i].cmd_str);

457 + cmd_ctx->args = lstrip;

458 cmd_ctx->message = NULL;

459

460 assert(0 == uv_queue_work(loop, &cmd_ctx->work, schedule_command, after_schedule_command));

daemon/commands.h

@@ -21,6 +21,8 @@ typedef enum cmd {

21 CMD_FATAL,

22 CMD_RELOAD_CLAIMING_STATE,

23 CMD_RELOAD_LABELS,

24 + CMD_READ_CONFIG,

25 + CMD_WRITE_CONFIG,

26 CMD_TOTAL_COMMANDS

27 } cmd_t;

daemon/daemon.c

+1 -1

@@ -437,7 +437,7 @@ int become_daemon(int dont_fork, const char *user)

437 sched_setscheduler_set();

438

439 // Set claiming directory based on user config directory with correct ownership

440 - snprintfz(claimingdirectory, FILENAME_MAX, "%s/claim.d", netdata_configured_user_config_dir);

440 + snprintfz(claimingdirectory, FILENAME_MAX, "%s/cloud.d", netdata_configured_varlib_dir);

441

442 if(user && *user) {

443 if(become_user(user, pidfd) != 0) {

daemon/main.c

+82 -37

@@ -577,20 +577,7 @@ static void get_netdata_configured_variables() {

577 get_system_cpus();

578 get_system_pid_max();

579

580 - // --------------------------------------------------------------------

581 - // Check if the cloud is enabled

582 -#ifdef DISABLE_CLOUD

583 - netdata_cloud_setting = 0;

584 -#else

585 - char *cloud = config_get(CONFIG_SECTION_GLOBAL, "netdata cloud", "coming soon");

586 - if (!strcmp(cloud, "coming soon")) {

587 - netdata_cloud_setting = 0; // Note: this flips to 1 after the release

588 - } else if (!strcmp(cloud, "enable")) {

589 - netdata_cloud_setting = 1;

590 - } else if (!strcmp(cloud, "disable")) {

591 - netdata_cloud_setting = 0;

592 - }

593 -#endif

580 +

581 }

582

583 static void get_system_timezone(void) {

@@ -851,11 +838,41 @@ void set_silencers_filename() {

838 silencers_filename = config_get(CONFIG_SECTION_HEALTH, "silencers file", filename);

839 }

840

841 +/* Any config setting that can be accessed without a default value i.e. configget(...,...,NULL) *MUST*

842 + be set in this procedure to be called in all the relevant code paths.

843 +*/

844 +void post_conf_load(char **user)

845 +{

846 + // --------------------------------------------------------------------

847 + // get the user we should run

848 +

849 + // IMPORTANT: this is required before web_files_uid()

850 + if(getuid() == 0) {

851 + *user = config_get(CONFIG_SECTION_GLOBAL, "run as user", NETDATA_USER);

852 + }

853 + else {

854 + struct passwd *passwd = getpwuid(getuid());

855 + *user = config_get(CONFIG_SECTION_GLOBAL, "run as user", (passwd && passwd->pw_name)?passwd->pw_name:"");

856 + }

857 +

858 + // --------------------------------------------------------------------

859 + // Check if the cloud is enabled

860 +#if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK )

861 + netdata_cloud_setting = 0;

862 +#else

863 + netdata_cloud_setting = appconfig_get_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", 1);

864 +#endif

865 + // This must be set before any point in the code that accesses it. Do not move it from this function.

866 + appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", DEFAULT_CLOUD_BASE_URL);

867 +}

868 +

869 int main(int argc, char **argv) {

870 int i;

871 int config_loaded = 0;

872 int dont_fork = 0;

873 size_t default_stacksize;

874 + char *user = NULL;

875 +

876

877 netdata_ready=0;

878 // set the name for logging

@@ -918,6 +935,8 @@ int main(int argc, char **argv) {

935 }

936 else {

937 debug(D_OPTIONS, "Configuration loaded from %s.", optarg);

938 + post_conf_load(&user);

939 + load_cloud_conf(1);

940 config_loaded = 1;

941 }

942 break;

@@ -965,6 +984,8 @@ int main(int argc, char **argv) {

984 if(strcmp(optarg, "unittest") == 0) {

985 if(unit_test_buffer()) return 1;

986 if(unit_test_str2ld()) return 1;

987 + // No call to load the config file on this code-path

988 + post_conf_load(&user);

989 get_netdata_configured_variables();

990 default_rrd_update_every = 1;

991 default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;

@@ -1065,9 +1086,9 @@ int main(int argc, char **argv) {

1086 debug_flags = strtoull(optarg, NULL, 0);

1087 }

1088 else if(strcmp(optarg, "set") == 0) {

1068 - if(optind + 3 > argc) {

1069 - fprintf(stderr, "%s", "\nUSAGE: -W set 'section' 'key' 'value'\n\n"

1070 - " Overwrites settings of netdata.conf.\n"

1089 + if(optind + 4 > argc) {

1090 + fprintf(stderr, "%s", "\nUSAGE: -W set 'conf_file' 'section' 'key' 'value'\n\n"

1091 + " Overwrites settings of netdata.conf or cloud.conf\n"

1092 "\n"

1093 " These options interact with: -c netdata.conf\n"

1094 " If -c netdata.conf is given on the command line,\n"

@@ -1076,21 +1097,24 @@ int main(int argc, char **argv) {

1097 " If -c netdata.conf is given after (or missing)\n"

1098 " -W set... the user cannot overwrite the command line\n"

1099 " parameters."

1100 + " conf_file can be \"cloud\" or \"netdata\".\n"

1101 "\n"

1102 );

1103 return 1;

1104 }

1083 - const char *section = argv[optind];

1084 - const char *key = argv[optind + 1];

1085 - const char *value = argv[optind + 2];

1086 - optind += 3;

1105 + const char *conf_file = argv[optind]; /* "cloud" is cloud.conf, otherwise netdata.conf */

1106 + struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config;

1107 + const char *section = argv[optind + 1];

1108 + const char *key = argv[optind + 2];

1109 + const char *value = argv[optind + 3];

1110 + optind += 4;

1111

1112 // set this one as the default

1113 // only if it is not already set in the config file

1114 // so the caller can use -c netdata.conf before or

1115 // after this parameter to prevent or allow overwriting

1116 // variables at netdata.conf

1093 - config_set_default(section, key, value);

1117 + appconfig_set_default(tmp_config, section, key, value);

1118

1119 // fprintf(stderr, "SET section '%s', key '%s', value '%s'\n", section, key, value);

1120 }

@@ -1109,6 +1133,7 @@ int main(int argc, char **argv) {

1133 if(!config_loaded) {

1134 fprintf(stderr, "warning: no configuration file has been loaded. Use -c CONFIG_FILE, before -W get. Using default config.\n");

1135 load_netdata_conf(NULL, 0);

1136 + post_conf_load(&user);

1137 }

1138

1139 get_netdata_configured_variables();

@@ -1120,6 +1145,37 @@ int main(int argc, char **argv) {

1145 printf("%s\n", value);

1146 return 0;

1147 }

1148 + else if(strcmp(optarg, "get2") == 0) {

1149 + if(optind + 4 > argc) {

1150 + fprintf(stderr, "%s", "\nUSAGE: -W get2 'conf_file' 'section' 'key' 'value'\n\n"

1151 + " Prints settings of netdata.conf or cloud.conf\n"

1152 + "\n"

1153 + " These options interact with: -c netdata.conf\n"

1154 + " -c netdata.conf has to be given before -W get2.\n"

1155 + " conf_file can be \"cloud\" or \"netdata\".\n"

1156 + "\n"

1157 + );

1158 + return 1;

1159 + }

1160 +

1161 + if(!config_loaded) {

1162 + fprintf(stderr, "warning: no configuration file has been loaded. Use -c CONFIG_FILE, before -W get. Using default config.\n");

1163 + load_netdata_conf(NULL, 0);

1164 + post_conf_load(&user);

1165 + load_cloud_conf(1);

1166 + }

1167 +

1168 + get_netdata_configured_variables();

1169 +

1170 + const char *conf_file = argv[optind]; /* "cloud" is cloud.conf, otherwise netdata.conf */

1171 + struct config *tmp_config = strcmp(conf_file, "cloud") ? &netdata_config : &cloud_config;

1172 + const char *section = argv[optind + 1];

1173 + const char *key = argv[optind + 2];

1174 + const char *def = argv[optind + 3];

1175 + const char *value = appconfig_get(tmp_config, section, key, def);

1176 + printf("%s\n", value);

1177 + return 0;

1178 + }

1179 else if(strncmp(optarg, claim_string, strlen(claim_string)) == 0) {

1180 /* will trigger a claiming attempt when the agent is initialized */

1181 claiming_pending_arguments = optarg + strlen(claim_string);

@@ -1149,7 +1205,11 @@ int main(int argc, char **argv) {

1205 #endif

1206

1207 if(!config_loaded)

1208 + {

1209 load_netdata_conf(NULL, 0);

1210 + post_conf_load(&user);

1211 + load_cloud_conf(0);

1212 + }

1213

1214

1215 // ------------------------------------------------------------------------

@@ -1179,8 +1239,6 @@ int main(int argc, char **argv) {

1239 fatal("Cannot cd to '%s'", netdata_configured_user_config_dir);

1240 }

1241

1182 - char *user = NULL;

1183 -

1242 {

1243 // --------------------------------------------------------------------

1244 // get the debugging flags from the configuration file

@@ -1246,19 +1304,6 @@ int main(int argc, char **argv) {

1304 }

1305

1306

1249 - // --------------------------------------------------------------------

1250 - // get the user we should run

1251 -

1252 - // IMPORTANT: this is required before web_files_uid()

1253 - if(getuid() == 0) {

1254 - user = config_get(CONFIG_SECTION_GLOBAL, "run as user", NETDATA_USER);

1255 - }

1256 - else {

1257 - struct passwd *passwd = getpwuid(getuid());

1258 - user = config_get(CONFIG_SECTION_GLOBAL, "run as user", (passwd && passwd->pw_name)?passwd->pw_name:"");

1259 - }

1260 -

1261 -

1307 // --------------------------------------------------------------------

1308 // create the listening sockets

1309

health/health_json.c

+25

@@ -341,3 +341,28 @@ void health_alarms_values2json(RRDHOST *host, BUFFER *wb, int all) {

341 buffer_strcat(wb, "\n\t}\n}\n");

342 rrdhost_unlock(host);

343 }

344 +

345 +

346 +void health_active_log_alarms_2json(RRDHOST *host, BUFFER *wb) {

347 + netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);

348 +

349 + buffer_sprintf(wb, "[\n");

350 +

351 + unsigned int max = host->health_log.max;

352 + unsigned int count = 0;

353 + ALARM_ENTRY *ae;

354 + for(ae = host->health_log.alarms; ae && count < max ; ae = ae->next) {

355 +

356 + if(likely(!((ae->new_status == RRDCALC_STATUS_WARNING || ae->new_status == RRDCALC_STATUS_CRITICAL)

357 + && (ae->old_status != RRDCALC_STATUS_WARNING || ae->old_status != RRDCALC_STATUS_CRITICAL)

358 + && !ae->updated_by_id)))

359 + continue;

360 +

361 + if(likely(count)) buffer_strcat(wb, ",");

362 + health_alarm_entry2json_nolock(wb, ae, host);

363 + count++;

364 + }

365 + buffer_strcat(wb, "]");

366 +

367 + netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);

368 +}

health/notifications/alarm-notify.sh.in

+1 -1

@@ -195,7 +195,7 @@ fi

195 [ -z "${NETDATA_STOCK_CONFIG_DIR}" ] && NETDATA_STOCK_CONFIG_DIR="@libconfigdir_POST@"

196 [ -z "${NETDATA_CACHE_DIR}" ] && NETDATA_CACHE_DIR="@cachedir_POST@"

197 [ -z "${NETDATA_REGISTRY_URL}" ] && NETDATA_REGISTRY_URL="https://registry.my-netdata.io"

198 -[ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://netdata.cloud"

198 +[ -z "${NETDATA_REGISTRY_CLOUD_BASE_URL}" ] && NETDATA_REGISTRY_CLOUD_BASE_URL="https://app.netdata.cloud"

199

200 # -----------------------------------------------------------------------------

201 # parse command line parameters

libnetdata/config/appconfig.c

+14 -5

@@ -289,18 +289,21 @@ char *appconfig_get_by_section(struct section *co, const char *name, const char

289 {

290 struct config_option *cv;

291

292 + // Only calls internal to this file check for a NULL result and they do not supply a NULL arg.

293 + // External caller should treat NULL as an error case.

294 cv = appconfig_option_index_find(co, name, 0);

293 - if(!cv) {

295 + if (!cv) {

296 + if (!default_value) return NULL;

297 cv = appconfig_value_create(co, name, default_value);

295 - if(!cv) return NULL;

298 + if (!cv) return NULL;

299 }

300 cv->flags |= CONFIG_VALUE_USED;

301

302 if((cv->flags & CONFIG_VALUE_LOADED) || (cv->flags & CONFIG_VALUE_CHANGED)) {

303 // this is a loaded value from the config file

301 - // if it is different that the default, mark it

304 + // if it is different than the default, mark it

305 if(!(cv->flags & CONFIG_VALUE_CHECKED)) {

303 - if(strcmp(cv->value, default_value) != 0) cv->flags |= CONFIG_VALUE_CHANGED;

306 + if(default_value && strcmp(cv->value, default_value) != 0) cv->flags |= CONFIG_VALUE_CHANGED;

307 cv->flags |= CONFIG_VALUE_CHECKED;

308 }

309 }

@@ -308,11 +311,17 @@ char *appconfig_get_by_section(struct section *co, const char *name, const char

311 return(cv->value);

312 }

313

314 +

315 char *appconfig_get(struct config *root, const char *section, const char *name, const char *default_value)

316 {

313 - debug(D_CONFIG, "request to get config in section '%s', name '%s', default_value '%s'", section, name, default_value);

317 + if (default_value == NULL)

318 + debug(D_CONFIG, "request to get config in section '%s', name '%s' or fail", section, name);

319 + else

320 + debug(D_CONFIG, "request to get config in section '%s', name '%s', default_value '%s'", section, name, default_value);

321

322 struct section *co = appconfig_section_find(root, section);

323 + if (!co && !default_value)

324 + return NULL;

325 if(!co) co = appconfig_section_create(root, section);

326

327 return appconfig_get_by_section(co, name, default_value);

libnetdata/libnetdata.h

+1 -1

@@ -323,6 +323,6 @@ extern char *netdata_configured_host_prefix;

323 #include "string/utf8.h"

324

325 // BEWARE: Outside of the C code this also exists in alarm-notify.sh

326 -#define DEFAULT_CLOUD_BASE_URL "https://netdata.cloud"

326 +#define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"

327

328 #endif // NETDATA_LIB_H

netdata-installer.sh

+71 -1

@@ -296,6 +296,9 @@ while [ -n "${1}" ]; do

296 NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--enable-cloud/} --enable-cloud"

297 fi

298 ;;

299 + "--build-json-c")

300 + NETDATA_BUILD_JSON_C=1

301 + ;;

302 "--install")

303 NETDATA_PREFIX="${2}/netdata"

304 shift 1

@@ -621,6 +624,73 @@ bundle_libwebsockets() {

624

625 bundle_libwebsockets

626

627 +# -----------------------------------------------------------------------------

628 +

629 +build_jsonc() {

630 + pushd "${1}" > /dev/null || exit 1

631 + run env CFLAGS= CXXFLAGS= LDFLAGS= cmake -DBUILD_SHARED_LIBS=OFF .

632 + run env CFLAGS= CXXFLAGS= LDFLAGS= make

633 + popd > /dev/null || exit 1

634 +}

635 +

636 +copy_jsonc() {

637 + target_dir="${PWD}/externaldeps/jsonc"

638 +

639 + run mkdir -p "${target_dir}" "${target_dir}/json-c" || return 1

640 +

641 + run cp "${1}/libjson-c.a" "${target_dir}/libjson-c.a" || return 1

642 + run cp ${1}/*.h "${target_dir}/json-c" || return 1

643 +}

644 +

645 +bundle_jsonc() {

646 + # If --build-json-c flag or not json-c on system, then bundle our own json-c

647 + if [ -z "${NETDATA_BUILD_JSON_C}" ] && pkg-config json-c; then

648 + return 0

649 + fi

650 +

651 + if [ -z "$(command -v cmake)" ]; then

652 + run_failed "Could not find cmake, which is required to build JSON-C. The install process will continue, but Netdata Cloud support will be disabled."

653 + defer_error_highlighted "Could not find cmake, which is required to build JSON-C. The install process will continue, but Netdata Cloud support will be disabled."

654 + return 0

655 + fi

656 +

657 + progress "Prepare JSON-C"

658 +

659 + JSONC_PACKAGE_VERSION="$(cat packaging/jsonc.version)"

660 +

661 + tmp="$(mktemp -d -t netdata-jsonc-XXXXXX)"

662 + JSONC_PACKAGE_BASENAME="json-c-${JSONC_PACKAGE_VERSION}.tar.gz"

663 +

664 + if fetch_and_verify "jsonc" \

665 + "https://github.com/json-c/json-c/archive/${JSONC_PACKAGE_BASENAME}" \

666 + "${JSONC_PACKAGE_BASENAME}" \

667 + "${tmp}" \

668 + "${NETDATA_LOCAL_TARBALL_OVERRIDE_JSONC}"; then

669 + if run tar -xf "${tmp}/${JSONC_PACKAGE_BASENAME}" -C "${tmp}" &&

670 + build_jsonc "${tmp}/json-c-json-c-${JSONC_PACKAGE_VERSION}" &&

671 + copy_jsonc "${tmp}/json-c-json-c-${JSONC_PACKAGE_VERSION}" &&

672 + rm -rf "${tmp}"; then

673 + run_ok "JSON-C built and prepared."

674 + else

675 + run_failed "Failed to build JSON-C."

676 + if [ -n "${NETDATA_REQUIRE_CLOUD}" ]; then

677 + exit 1

678 + else

679 + defer_error_highlighted "Failed to build JSON-C. Netdata Cloud support will be disabled."

680 + fi

681 + fi

682 + else

683 + run_failed "Unable to fetch sources for JSON-C."

684 + if [ -n "${NETDATA_REQUIRE_CLOUD}" ]; then

685 + exit 1

686 + else

687 + defer_error_highlighted "Unable to fetch sources for JSON-C. Netdata Cloud support will be disabled."

688 + fi

689 + fi

690 +}

691 +

692 +bundle_jsonc

693 +

694 # -----------------------------------------------------------------------------

695 # If we have the dashboard switching logic, make sure we're on the classic

696 # dashboard during the install (updates don't work correctly otherwise).

@@ -853,7 +923,7 @@ NETDATA_LOG_DIR="$(config_option "global" "log directory" "${NETDATA_PREFIX}/var

923 NETDATA_USER_CONFIG_DIR="$(config_option "global" "config directory" "${NETDATA_PREFIX}/etc/netdata")"

924 NETDATA_STOCK_CONFIG_DIR="$(config_option "global" "stock config directory" "${NETDATA_PREFIX}/usr/lib/netdata/conf.d")"

925 NETDATA_RUN_DIR="${NETDATA_PREFIX}/var/run"

856 -NETDATA_CLAIMING_DIR="${NETDATA_USER_CONFIG_DIR}/claim.d"

926 +NETDATA_CLAIMING_DIR="${NETDATA_LIB_DIR}/cloud.d"

927

928 cat << OPTIONSEOF

929

packaging/dashboard.checksums

+1 -1

@@ -1 +1 @@

1 -aa0056df0e79720cf9bda68a9e4051cd021f40c482f64bfa46645cdd1243b93e dashboard.tar.gz

1 +c48c971cef360a08ac82ae43c7f50fdce32da4b0fc78b0f67b296fa154dfd7a6 dashboard.tar.gz

packaging/dashboard.version

+1 -1

@@ -1 +1 @@

1 -v0.4.18

1 +v1.0.14

packaging/docker/Dockerfile

+2 -2

@@ -98,8 +98,8 @@ RUN \

98 /var/cache/netdata \

99 /var/lib/netdata \

100 /var/log/netdata && \

101 - chown -R netdata:netdata /etc/netdata/claim.d && \

102 - chmod 0700 /etc/netdata/claim.d && \

101 + chown -R netdata:netdata /var/lib/netdata/cloud.d && \

102 + chmod 0700 /var/lib/netdata/cloud.d && \

103 chmod 0755 /usr/libexec/netdata/plugins.d/*.plugin && \

104 chmod 4755 \

105 /usr/libexec/netdata/plugins.d/cgroup-network \

packaging/jsonc.checksums new

@@ -0,0 +1 @@

1 +ec4eb70e0f6c0d707b9b1ec646cf7c860f4abb3562a90ea6e4d78d177fd95303 json-c-0.14-20200419.tar.gz

packaging/jsonc.version new

@@ -0,0 +1 @@

1 +0.14-20200419

registry/registry.c

+14

@@ -129,6 +129,20 @@ static inline int registry_person_url_callback_verify_machine_exists(void *entry

129 return 0;

130 }

131

132 +// ----------------------------------------------------------------------------

133 +// dynamic update of the configuration

134 +// The registry does not seem to be designed to support this and I cannot see any concurrency protection

135 +// that could make this safe, so try to be as atomic as possible.

136 +

137 +void registry_update_cloud_base_url()

138 +{

139 + // This is guaranteed to be set early in main via post_conf_load()

140 + registry.cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL);

141 + if (registry.cloud_base_url == NULL)

142 + fatal("Do not move the cloud base url out of post_conf_load!!");

143 +

144 + setenv("NETDATA_REGISTRY_CLOUD_BASE_URL", registry.cloud_base_url, 1);

145 +}

146 // ----------------------------------------------------------------------------

147 // public HELLO request

148

registry/registry.h

@@ -68,6 +68,9 @@ extern int registry_request_search_json(RRDHOST *host, struct web_client *w, cha

68 extern int registry_request_switch_json(RRDHOST *host, struct web_client *w, char *person_guid, char *machine_guid, char *url, char *new_person_guid, time_t when);

69 extern int registry_request_hello_json(RRDHOST *host, struct web_client *w);

71 +// update the registry config

72 +extern void registry_update_cloud_base_url();

73 +

74 // update the registry monitoring charts

75 extern void registry_statistics(void);

registry/registry_init.c

+1 -4

@@ -40,10 +40,7 @@ int registry_init(void) {

40 registry.hostname = config_get(CONFIG_SECTION_REGISTRY, "registry hostname", netdata_configured_hostname);

41 registry.verify_cookies_redirects = config_get_boolean(CONFIG_SECTION_REGISTRY, "verify browser cookies support", 1);

43 - // netdata.cloud configuration, if cloud_base_url == "", cloud functionality is disabled.

44 - registry.cloud_base_url = config_get(CONFIG_SECTION_CLOUD, "cloud base url", DEFAULT_CLOUD_BASE_URL);

45 -

46 - setenv("NETDATA_REGISTRY_CLOUD_BASE_URL", registry.cloud_base_url, 1);

43 + registry_update_cloud_base_url();

44 setenv("NETDATA_REGISTRY_HOSTNAME", registry.hostname, 1);

45 setenv("NETDATA_REGISTRY_URL", registry.registry_to_announce, 1);

tests/alarm_repetition/netdata.conf_with_repetition

+1 -1

@@ -54,4 +54,4 @@

54 allow from = *

56 [cloud]

57 - cloud base url = https://netdata.cloud

57 + cloud base url = https://app.netdata.cloud

tests/alarm_repetition/netdata.conf_without_repetition

+1 -1

@@ -54,4 +54,4 @@

54 allow from = *

56 [cloud]

57 - cloud base url = https://netdata.cloud

57 + cloud base url = https://app.netdata.cloud

web/api/web_api_v1.c

+3 -4

@@ -865,11 +865,10 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)

865 #ifdef DISABLE_CLOUD

866 buffer_strcat(wb, "\t\"cloud-enabled\": false,\n");

867 #else

868 - if (netdata_cloud_setting)

869 - buffer_strcat(wb, "\t\"cloud-enabled\": true,\n");

870 - else

871 - buffer_strcat(wb, "\t\"cloud-enabled\": false,\n");

868 + buffer_sprintf(wb, "\t\"cloud-enabled\": %s,\n",

869 + appconfig_get_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", 1) ? "true" : "false");

870 #endif

871 +

872 #ifdef ENABLE_ACLK

873 buffer_strcat(wb, "\t\"cloud-available\": true,\n");

874 #else

web/gui/main.js

+1 -49

@@ -792,11 +792,6 @@ function renderMyNetdataMenu(machinesArray) {

792 if (!isSignedIn()) {

793 html += (

794 `<div class="agent-item">

795 - <i class="fas fa-tv"></i>

796 - <a onClick="openAuthenticatedUrl('console.html');" target="_blank">Nodes<sup class="beta"> beta</sup></a>

797 - <div></div>

798 - </div>

799 - <div class="agent-item">

795 <i class="fas fa-cog""></i>

796 <a href="#" onclick="switchRegistryModalHandler(); return false;">Switch Identity</a>

797 <div></div>

@@ -4807,11 +4802,7 @@ function signInDidClick(e) {

4802 }

4803

4804 function shouldShowSignInBanner() {

4810 - if (isSignedIn()) {

4811 - return false;

4812 - }

4813 -

4814 - return localStorage.getItem("signInBannerClosed") != "true";

4805 + return false;

4806 }

4807

4808 function closeSignInBanner() {

@@ -4895,43 +4886,6 @@ function signOut() {

4886 cloudSSOSignOut();

4887 }

4888

4898 -function renderAccountUI() {

4899 - if (!NETDATA.registry.isCloudEnabled) {

4900 - return

4901 - }

4902 -

4903 - const container = document.getElementById("account-menu-container");

4904 - if (isSignedIn()) {

4905 - container.removeAttribute("title");

4906 - container.removeAttribute("data-original-title");

4907 - container.removeAttribute("data-placement");

4908 - container.innerHTML = (

4909 - `<a href="#" class="dropdown-toggle" data-toggle="dropdown"><span id="amc-account-name"></span> <strong class="caret"></strong></a>

4910 - <ul id="cloud-menu" class="dropdown-menu scrollable-menu inpagemenu" role="menu">

4911 - <li>

4912 - <a onclick="openAuthenticatedUrl('console.html');" target="_blank" class="btn">

4913 - <i class="fas fa-tv"></i>  <span class="hidden-sm hidden-md">Nodes<sup class="beta"> beta</sup></span>

4914 - </a>

4915 - </li>

4916 - <li>

4917 - <a href="#" class="btn" onclick="signOutDidClick(event); return false">

4918 - <i class="fas fa-sign-out-alt"></i>  <span class="hidden-sm hidden-md">Sign Out</span>

4919 - </a>

4920 - </li>

4921 - </ul>`

4922 - )

4923 - document.getElementById("amc-account-name").textContent = cloudAccountName; // Anti-XSS

4924 - } else {

4925 - container.setAttribute("data-original-title", "sign in");

4926 - container.setAttribute("data-placement", "bottom");

4927 - container.innerHTML = (

4928 - `<a href="#" class="btn sign-in-btn theme-${netdataTheme}" onclick="signInDidClick(event); return false">

4929 - <i class="fas fa-sign-in-alt"></i> <span class="hidden-sm hidden-md">Sign In</span>

4930 - </a>`

4931 - )

4932 - }

4933 -}

4934 -

4889 function handleMessage(e) {

4890 switch (e.data.type) {

4891 case "sign-in":

@@ -4964,7 +4918,6 @@ function handleSignInMessage(e) {

4918

4919 function handleSignOutMessage(e) {

4920 clearCloudVariables();

4967 - renderAccountUI();

4921 renderMyNetdataMenu(registryAgents);

4922 }

4923

@@ -5118,7 +5071,6 @@ function initCloud() {

5071 }

5072

5073 touchAgent();

5121 - renderAccountUI();

5074 }

5075

5076 // This callback is called after NETDATA.registry is initialized.

web/gui/old/index.html

+1 -2

@@ -33,7 +33,7 @@

33 <meta name="twitter:description" content="Unparalleled insights, in real-time, of everything happening on your Linux systems and applications, with stunning, interactive web dashboards and powerful performance and health alarms." />

34 <meta name="twitter:image" content="https://cloud.githubusercontent.com/assets/2662304/14092712/93b039ea-f551-11e5-822c-beadbf2b2a2e.gif" />

36 - <script src="../main.js?v20190905-0"></script>

36 + <script src="../main.js?v20200429-0"></script>

37 </head>

39 <body data-spy="scroll" data-target="#sidebar" data-offset="100">

@@ -90,7 +90,6 @@

90 </div>

91 <nav class="collapse navbar-collapse navbar-right" role="navigation">

92 <ul class="nav navbar-nav">

93 - <li title="Nodes view" data-toggle="tooltip" data-placement="bottom"><a onclick="openAuthenticatedUrl('console.html');" class="btn" target="_blank"><i class="fas fa-tv"></i> <span class="hidden-sm hidden-md">Nodes<sup class="beta"> beta</sup></span></a></li>

93 <li id="alarmsButton" title="check the health monitoring alarms and their log" data-toggle="tooltip" data-placement="bottom"><a href="#" class="btn" data-toggle="modal" data-target="#alarmsModal"><i class="fas fa-bell"></i> <span class="hidden-sm hidden-md">Alarms </span><span id="alarms_count_badge" class="badge"></span></a></li>

94 <li title="change dashboard settings" data-toggle="tooltip" data-placement="bottom"><a href="#" class="btn" data-toggle="modal" data-target="#optionsModal"><i class="fas fa-cog"></i> <span class="hidden-sm hidden-md">Settings</span></a></li>

95 <li title="check for netdata updates<br/>you should keep your netdata updated" data-toggle="tooltip" data-placement="bottom" class="hidden-sm" id="updateButton"><a href="#" class="btn" data-toggle="modal" data-target="#updateModal"><i class="fas fa-cloud-download-alt"></i> <span class="hidden-sm hidden-md">Update </span><span id="update_badge" class="badge"></span></a></li>