@cryptotaxi247 / netdata-1 / commits / 52456f5ba

Remove backends subsystem (#12146)

Vladimir Kobal committed Mar 15, 2022 at 11:50 UTC 52456f5baf6e677c7e2cf5fcb863c59076e01093

88 files changed +211 -5248

.github/CODEOWNERS

-5

@@ -8,11 +8,6 @@

8 .travis/ @Ferroin @iigorkarpov @maneamarius @kaskavel

9 .github/ @Ferroin @iigorkarpov @maneamarius @kaskavel

10 aclk/ @stelfrag @underhood

11 -backends/ @thiagoftsm @vlvkobal

12 -backends/graphite/ @thiagoftsm @vlvkobal

13 -backends/json/ @thiagoftsm @vlvkobal

14 -backends/opentsdb/ @thiagoftsm @vlvkobal

15 -backends/prometheus/ @vlvkobal @thiagoftsm

11 build/ @Ferroin @iigorkarpov @maneamarius

12 contrib/debian @Ferroin @iigorkarpov @maneamarius

13 collectors/ @vlvkobal

.github/labeler.yml

-4

@@ -15,10 +15,6 @@ ACLK:

15 - aclk/**/*

16 - mqtt_websockets

18 -area/backends:

19 - - backends/*

20 - - backends/**/*

21 -

18 area/claim:

19 - claim/*

CMakeLists.txt

+21 -52

@@ -745,19 +745,6 @@ set(STREAMING_PLUGIN_FILES

745 streaming/sender.c

746 )

747

748 -set(BACKENDS_PLUGIN_FILES

749 - backends/backends.c

750 - backends/backends.h

751 - backends/graphite/graphite.c

752 - backends/graphite/graphite.h

753 - backends/json/json.c

754 - backends/json/json.h

755 - backends/opentsdb/opentsdb.c

756 - backends/opentsdb/opentsdb.h

757 - backends/prometheus/backend_prometheus.c

758 - backends/prometheus/backend_prometheus.h

759 - )

760 -

748 set(CLAIM_PLUGIN_FILES

749 claim/claim.c

750 claim/claim.h

@@ -884,23 +871,6 @@ set(MONGODB_EXPORTING_FILES

871 exporting/mongodb/mongodb.h

872 )

873

887 -set(KINESIS_BACKEND_FILES

888 - backends/aws_kinesis/aws_kinesis.c

889 - backends/aws_kinesis/aws_kinesis.h

890 - backends/aws_kinesis/aws_kinesis_put_record.cc

891 - backends/aws_kinesis/aws_kinesis_put_record.h

892 - )

893 -

894 -set(PROMETHEUS_REMOTE_WRITE_BACKEND_FILES

895 - backends/prometheus/remote_write/remote_write.cc

896 - backends/prometheus/remote_write/remote_write.h

897 - )

898 -

899 -set(MONGODB_BACKEND_FILES

900 - backends/mongodb/mongodb.c

901 - backends/mongodb/mongodb.h

902 - )

903 -

874 set(DAEMON_FILES

875 daemon/buildinfo.c

876 daemon/buildinfo.h

@@ -960,7 +930,6 @@ set(NETDATA_FILES

930 collectors/all.h

931 ${DAEMON_FILES}

932 ${API_PLUGIN_FILES}

963 - ${BACKENDS_PLUGIN_FILES}

933 ${EXPORTING_ENGINE_FILES}

934 ${CHECKS_PLUGIN_FILES}

935 ${HEALTH_PLUGIN_FILES}

@@ -997,25 +966,25 @@ add_definitions(

966 )

967

968 # -----------------------------------------------------------------------------

1000 -# kinesis backend

969 +# kinesis exporting connector

970

971 IF(KINESIS_LIBRARIES AND AWS_CORE_LIBRARIES AND HAVE_AWS_EVENT_STREAM AND HAVE_AWS_COMMON AND HAVE_AWS_CHECKSUMS AND

972 CRYPTO_LIBRARIES AND SSL_LIBRARIES AND CURL_LIBRARIES)

1004 - SET(ENABLE_BACKEND_KINESIS True)

973 + SET(ENABLE_EXPORTING_KINESIS True)

974 ELSE()

1006 - SET(ENABLE_BACKEND_KINESIS False)

975 + SET(ENABLE_EXPORTING_KINESIS False)

976 ENDIF()

977

1009 -IF(ENABLE_BACKEND_KINESIS)

1010 - message(STATUS "kinesis backend: enabled")

1011 - list(APPEND NETDATA_FILES ${KINESIS_BACKEND_FILES} ${KINESIS_EXPORTING_FILES})

978 +IF(ENABLE_EXPORTING_KINESIS)

979 + message(STATUS "kinesis exporting: enabled")

980 + list(APPEND NETDATA_FILES ${KINESIS_EXPORTING_FILES})

981 list(APPEND NETDATA_COMMON_LIBRARIES ${KINESIS_LIBRARIES} ${AWS_CORE_LIBRARIES}

982 ${CRYPTO_LIBRARIES} ${SSL_LIBRARIES} ${CURL_LIBRARIES})

983 list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${KINESIS_INCLUDE_DIRS} ${AWS_CORE_INCLUDE_DIRS}

984 ${CRYPTO_INCLUDE_DIRS} ${SSL_INCLUDE_DIRS} ${CURL_INCLUDE_DIRS})

985 list(APPEND NETDATA_COMMON_CFLAGS ${CRYPTO_CFLAGS_OTHER} ${SSL_CFLAGS_OTHER} ${CURL_CFLAGS_OTHER})

986 ELSE()

1018 - message(STATUS "kinesis backend: disabled (requires AWS SDK for C++)")

987 + message(STATUS "kinesis exporting: disabled (requires AWS SDK for C++)")

988 ENDIF()

989

990 # -----------------------------------------------------------------------------

@@ -1038,16 +1007,16 @@ ELSE()

1007 ENDIF()

1008

1009 # -----------------------------------------------------------------------------

1041 -# prometheus remote write backend

1010 +# prometheus remote write exporting connector

1011

1012 IF(PROTOBUF_LIBRARIES AND SNAPPY_LIBRARIES)

1044 - SET(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE True)

1013 + SET(ENABLE_EXPORTING_PROMETHEUS_REMOTE_WRITE True)

1014 ELSE()

1046 - SET(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE False)

1015 + SET(ENABLE_EXPORTING_PROMETHEUS_REMOTE_WRITE False)

1016 ENDIF()

1017

1049 -IF(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)

1050 - message(STATUS "prometheus remote write backend: enabled")

1018 +IF(ENABLE_EXPORTING_PROMETHEUS_REMOTE_WRITE)

1019 + message(STATUS "prometheus remote write exporting: enabled")

1020

1021 find_package(Protobuf REQUIRED)

1022

@@ -1083,26 +1052,26 @@ IF(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)

1052

1053 protobuf_remote_write_generate_cpp(PROTO_SRCS PROTO_HDRS exporting/prometheus/remote_write/remote_write.proto)

1054

1086 - list(APPEND NETDATA_FILES ${PROMETHEUS_REMOTE_WRITE_BACKEND_FILES} ${PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES} ${PROTO_SRCS} ${PROTO_HDRS})

1055 + list(APPEND NETDATA_FILES ${PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES} ${PROTO_SRCS} ${PROTO_HDRS})

1056 list(APPEND NETDATA_COMMON_LIBRARIES ${PROTOBUF_LIBRARIES} ${SNAPPY_LIBRARIES})

1057 list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${PROTOBUF_INCLUDE_DIRS} ${SNAPPY_INCLUDE_DIRS} ${CMAKE_CURRENT_BINARY_DIR})

1058 list(APPEND NETDATA_COMMON_CFLAGS ${PROTOBUF_CFLAGS_OTHER} ${SNAPPY_CFLAGS_OTHER})

1059 ELSE()

1091 - message(STATUS "prometheus remote write backend: disabled (requires protobuf and snappy libraries)")

1060 + message(STATUS "prometheus remote write exporting: disabled (requires protobuf and snappy libraries)")

1061 ENDIF()

1062

1063 # -----------------------------------------------------------------------------

1095 -# mongodb backend

1064 +# mongodb exporting connector

1065

1066 IF(MONGOC_LIBRARIES)

1098 - message(STATUS "mongodb backend: enabled")

1067 + message(STATUS "mongodb exporting: enabled")

1068

1100 - list(APPEND NETDATA_FILES ${MONGODB_BACKEND_FILES} ${MONGODB_EXPORTING_FILES})

1069 + list(APPEND NETDATA_FILES ${MONGODB_EXPORTING_FILES})

1070 list(APPEND NETDATA_COMMON_LIBRARIES ${MONGOC_LIBRARIES})

1071 list(APPEND NETDATA_COMMON_INCLUDE_DIRS ${MONGOC_INCLUDE_DIRS})

1072 list(APPEND NETDATA_COMMON_CFLAGS ${MONGOC_CFLAGS_OTHER})

1073 ELSE()

1105 - message(STATUS "mongodb backend: disabled (requires mongoc library)")

1074 + message(STATUS "mongodb exporting: disabled (requires mongoc library)")

1075 ENDIF()

1076

1077 set(NETDATA_COMMON_LIBRARIES ${NETDATA_COMMON_LIBRARIES} m ${CMAKE_THREAD_LIBS_INIT})

@@ -1223,7 +1192,7 @@ ELSEIF(MACOS)

1192

1193 ENDIF()

1194

1226 -IF(ENABLE_BACKEND_KINESIS OR ENABLE_EXPORTING_PUBSUB OR ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)

1195 +IF(ENABLE_EXPORTING_KINESIS OR ENABLE_EXPORTING_PUBSUB OR ENABLE_EXPORTING_PROMETHEUS_REMOTE_WRITE)

1196 set_property(TARGET netdata PROPERTY CXX_STANDARD 11)

1197 set_property(TARGET netdata PROPERTY CMAKE_CXX_STANDARD_REQUIRED ON)

1198 ENDIF()

@@ -1397,7 +1366,7 @@ if(BUILD_TESTING)

1366 set(KINESIS_LINK_OPTIONS)

1367 set(PUBSUB_LINK_OPTIONS)

1368 set(MONGODB_LINK_OPTIONS)

1400 -if(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)

1369 +if(ENABLE_EXPORTING_PROMETHEUS_REMOTE_WRITE)

1370 list(APPEND EXPORTING_ENGINE_FILES ${PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES} ${PROTO_SRCS} ${PROTO_HDRS})

1371 list(

1372 APPEND PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS

@@ -1407,7 +1376,7 @@ if(ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE)

1376 -Wl,--wrap=add_metric

1377 )

1378 endif()

1410 -if(ENABLE_BACKEND_KINESIS)

1379 +if(ENABLE_EXPORTING_KINESIS)

1380 list(APPEND EXPORTING_ENGINE_FILES ${KINESIS_EXPORTING_FILES})

1381 list(

1382 APPEND KINESIS_LINK_OPTIONS

Makefile.am

+13 -45

@@ -99,7 +99,6 @@ dist_noinst_SCRIPTS = \

99 # Compile netdata binaries

100

101 SUBDIRS += \

102 - backends \

102 collectors \

103 daemon \

104 database \

@@ -580,19 +579,6 @@ WEB_PLUGIN_FILES = \

579 web/server/static/static-threaded.h \

580 $(NULL)

581

583 -BACKENDS_PLUGIN_FILES = \

584 - backends/backends.c \

585 - backends/backends.h \

586 - backends/graphite/graphite.c \

587 - backends/graphite/graphite.h \

588 - backends/json/json.c \

589 - backends/json/json.h \

590 - backends/opentsdb/opentsdb.c \

591 - backends/opentsdb/opentsdb.h \

592 - backends/prometheus/backend_prometheus.c \

593 - backends/prometheus/backend_prometheus.h \

594 - $(NULL)

595 -

582 CLAIM_FILES = \

583 claim/claim.c \

584 claim/claim.h \

@@ -831,23 +817,6 @@ MONGODB_EXPORTING_FILES = \

817 exporting/mongodb/mongodb.h \

818 $(NULL)

819

834 -KINESIS_BACKEND_FILES = \

835 - backends/aws_kinesis/aws_kinesis.c \

836 - backends/aws_kinesis/aws_kinesis.h \

837 - backends/aws_kinesis/aws_kinesis_put_record.cc \

838 - backends/aws_kinesis/aws_kinesis_put_record.h \

839 - $(NULL)

840 -

841 -PROMETHEUS_REMOTE_WRITE_BACKEND_FILES = \

842 - backends/prometheus/remote_write/remote_write.cc \

843 - backends/prometheus/remote_write/remote_write.h \

844 - $(NULL)

845 -

846 -MONGODB_BACKEND_FILES = \

847 - backends/mongodb/mongodb.c \

848 - backends/mongodb/mongodb.h \

849 - $(NULL)

850 -

820 DAEMON_FILES = \

821 daemon/buildinfo.c \

822 daemon/buildinfo.h \

@@ -877,7 +846,6 @@ NETDATA_FILES = \

846 $(DAEMON_FILES) \

847 $(LIBNETDATA_FILES) \

848 $(API_PLUGIN_FILES) \

880 - $(BACKENDS_PLUGIN_FILES) \

849 $(EXPORTING_ENGINE_FILES) \

850 $(CHECKS_PLUGIN_FILES) \

851 $(HEALTH_PLUGIN_FILES) \

@@ -1065,8 +1033,8 @@ if ENABLE_PLUGIN_SLABINFO

1033 $(NULL)

1034 endif

1035

1068 -if ENABLE_BACKEND_KINESIS

1069 - netdata_SOURCES += $(KINESIS_BACKEND_FILES) $(KINESIS_EXPORTING_FILES)

1036 +if ENABLE_EXPORTING_KINESIS

1037 + netdata_SOURCES += $(KINESIS_EXPORTING_FILES)

1038 netdata_LDADD += $(OPTIONAL_KINESIS_LIBS)

1039 endif

1040

@@ -1075,17 +1043,17 @@ if ENABLE_EXPORTING_PUBSUB

1043 netdata_LDADD += $(OPTIONAL_PUBSUB_LIBS)

1044 endif

1045

1078 -if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE

1079 - netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_BACKEND_FILES) $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)

1046 +if ENABLE_EXPORTING_PROMETHEUS_REMOTE_WRITE

1047 + netdata_SOURCES += $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)

1048 netdata_LDADD += $(OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS) \

1049 $(OPTIONAL_PROTOBUF_LIBS) \

1050 $(NULL)

1083 - BACKEND_PROMETHEUS_BUILT_SOURCES = \

1051 + EXPORTING_PROMETHEUS_BUILT_SOURCES = \

1052 exporting/prometheus/remote_write/remote_write.pb.cc \

1053 exporting/prometheus/remote_write/remote_write.pb.h \

1054 $(NULL)

1087 - BUILT_SOURCES += $(BACKEND_PROMETHEUS_BUILT_SOURCES)

1088 - nodist_netdata_SOURCES += $(BACKEND_PROMETHEUS_BUILT_SOURCES)

1055 + BUILT_SOURCES += $(EXPORTING_PROMETHEUS_BUILT_SOURCES)

1056 + nodist_netdata_SOURCES += $(EXPORTING_PROMETHEUS_BUILT_SOURCES)

1057

1058 exporting/prometheus/remote_write/remote_write.pb.cc \

1059 exporting/prometheus/remote_write/remote_write.pb.h: exporting/prometheus/remote_write/remote_write.proto

@@ -1093,8 +1061,8 @@ exporting/prometheus/remote_write/remote_write.pb.h: exporting/prometheus/remote

1061

1062 endif

1063

1096 -if ENABLE_BACKEND_MONGODB

1097 - netdata_SOURCES += $(MONGODB_BACKEND_FILES) $(MONGODB_EXPORTING_FILES)

1064 +if ENABLE_EXPORTING_MONGODB

1065 + netdata_SOURCES += $(MONGODB_EXPORTING_FILES)

1066 netdata_LDADD += $(OPTIONAL_MONGOC_LIBS)

1067 endif

1068

@@ -1217,7 +1185,7 @@ if ENABLE_UNITTESTS

1185 $(TEST_LDFLAGS) \

1186 $(NULL)

1187 exporting_tests_exporting_engine_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)

1220 -if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE

1188 +if ENABLE_EXPORTING_PROMETHEUS_REMOTE_WRITE

1189 exporting_tests_exporting_engine_testdriver_SOURCES += $(PROMETHEUS_REMOTE_WRITE_EXPORTING_FILES)

1190 exporting_tests_exporting_engine_testdriver_LDADD += \

1191 $(OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS) \

@@ -1229,9 +1197,9 @@ if ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE

1197 -Wl,--wrap=add_label \

1198 -Wl,--wrap=add_metric \

1199 $(NULL)

1232 - nodist_exporting_tests_exporting_engine_testdriver_SOURCES = $(BACKEND_PROMETHEUS_BUILT_SOURCES)

1200 + nodist_exporting_tests_exporting_engine_testdriver_SOURCES = $(EXPORTING_PROMETHEUS_BUILT_SOURCES)

1201 endif

1234 -if ENABLE_BACKEND_KINESIS

1202 +if ENABLE_EXPORTING_KINESIS

1203 exporting_tests_exporting_engine_testdriver_SOURCES += $(KINESIS_EXPORTING_FILES)

1204 exporting_tests_exporting_engine_testdriver_LDADD += $(OPTIONAL_KINESIS_LIBS)

1205 exporting_tests_exporting_engine_testdriver_LDFLAGS += \

@@ -1251,7 +1219,7 @@ if ENABLE_EXPORTING_PUBSUB

1219 -Wl,--wrap=pubsub_get_result \

1220 $(NULL)

1221 endif

1254 -if ENABLE_BACKEND_MONGODB

1222 +if ENABLE_EXPORTING_MONGODB

1223 exporting_tests_exporting_engine_testdriver_SOURCES += $(MONGODB_EXPORTING_FILES)

1224 exporting_tests_exporting_engine_testdriver_LDADD += $(OPTIONAL_MONGOC_LIBS)

1225 exporting_tests_exporting_engine_testdriver_LDFLAGS += \

backends/Makefile.am deleted

-22

@@ -1,22 +0,0 @@

1 -# SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -AUTOMAKE_OPTIONS = subdir-objects

4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in

5 -

6 -SUBDIRS = \

7 - graphite \

8 - json \

9 - opentsdb \

10 - prometheus \

11 - aws_kinesis \

12 - mongodb \

13 - $(NULL)

14 -

15 -dist_noinst_DATA = \

16 - README.md \

17 - WALKTHROUGH.md \

18 - $(NULL)

19 -

20 -dist_noinst_SCRIPTS = \

21 - nc-backend.sh \

22 - $(NULL)

backends/README.md deleted

-236

@@ -1,236 +0,0 @@

1 -<!--

2 -title: "Metrics long term archiving"

3 -custom_edit_url: https://github.com/netdata/netdata/edit/master/backends/README.md

4 --->

5 -

6 -# Metrics long term archiving

7 -

8 -> ⚠️ The backends system is now deprecated in favor of the [exporting engine](/exporting/README.md).

9 -

10 -Netdata supports backends for archiving the metrics, or providing long term dashboards, using Grafana or other tools,

11 -like this:

12 -

13 -![image](https://cloud.githubusercontent.com/assets/2662304/20649711/29f182ba-b4ce-11e6-97c8-ab2c0ab59833.png)

14 -

15 -Since Netdata collects thousands of metrics per server per second, which would easily congest any backend server when

16 -several Netdata servers are sending data to it, Netdata allows sending metrics at a lower frequency, by resampling them.

17 -

18 -So, although Netdata collects metrics every second, it can send to the backend servers averages or sums every X seconds

19 -(though, it can send them per second if you need it to).

20 -

21 -## features

22 -

23 -1. Supported backends

24 -

25 - - **graphite** (`plaintext interface`, used by **Graphite**, **InfluxDB**, **KairosDB**, **Blueflood**,

26 - **ElasticSearch** via logstash tcp input and the graphite codec, etc)

27 -

28 - metrics are sent to the backend server as `prefix.hostname.chart.dimension`. `prefix` is configured below,

29 - `hostname` is the hostname of the machine (can also be configured).

30 -

31 - - **opentsdb** (`telnet or HTTP interfaces`, used by **OpenTSDB**, **InfluxDB**, **KairosDB**, etc)

32 -

33 - metrics are sent to opentsdb as `prefix.chart.dimension` with tag `host=hostname`.

34 -

35 - - **json** document DBs

36 -

37 - metrics are sent to a document db, `JSON` formatted.

38 -

39 - - **prometheus** is described at [prometheus page](/backends/prometheus/README.md) since it pulls data from

40 - Netdata.

41 -

42 - - **prometheus remote write** (a binary snappy-compressed protocol buffer encoding over HTTP used by

43 - **Elasticsearch**, **Gnocchi**, **Graphite**, **InfluxDB**, **Kafka**, **OpenTSDB**, **PostgreSQL/TimescaleDB**,

44 - **Splunk**, **VictoriaMetrics**, and a lot of other [storage

45 - providers](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage))

46 -

47 - metrics are labeled in the format, which is used by Netdata for the [plaintext prometheus

48 - protocol](/backends/prometheus/README.md). Notes on using the remote write backend are [here](/backends/prometheus/remote_write/README.md).

49 -

50 - - **TimescaleDB** via [community-built connector](/backends/TIMESCALE.md) that takes JSON streams from a Netdata

51 - client and writes them to a TimescaleDB table.

52 -

53 - - **AWS Kinesis Data Streams**

54 -

55 - metrics are sent to the service in `JSON` format.

56 -

57 - - **MongoDB**

58 -

59 - metrics are sent to the database in `JSON` format.

60 -

61 -2. Only one backend may be active at a time.

62 -

63 -3. Netdata can filter metrics (at the chart level), to send only a subset of the collected metrics.

64 -

65 -4. Netdata supports three modes of operation for all backends:

66 -

67 - - `as-collected` sends to backends the metrics as they are collected, in the units they are collected. So,

68 - counters are sent as counters and gauges are sent as gauges, much like all data collectors do. For example, to

69 - calculate CPU utilization in this format, you need to know how to convert kernel ticks to percentage.

70 -

71 - - `average` sends to backends normalized metrics from the Netdata database. In this mode, all metrics are sent as

72 - gauges, in the units Netdata uses. This abstracts data collection and simplifies visualization, but you will not

73 - be able to copy and paste queries from other sources to convert units. For example, CPU utilization percentage

74 - is calculated by Netdata, so Netdata will convert ticks to percentage and send the average percentage to the

75 - backend.

76 -

77 - - `sum` or `volume`: the sum of the interpolated values shown on the Netdata graphs is sent to the backend. So, if

78 - Netdata is configured to send data to the backend every 10 seconds, the sum of the 10 values shown on the

79 - Netdata charts will be used.

80 -

81 - Time-series databases suggest to collect the raw values (`as-collected`). If you plan to invest on building your

82 - monitoring around a time-series database and you already know (or you will invest in learning) how to convert units

83 - and normalize the metrics in Grafana or other visualization tools, we suggest to use `as-collected`.

84 -

85 - If, on the other hand, you just need long term archiving of Netdata metrics and you plan to mainly work with

86 - Netdata, we suggest to use `average`. It decouples visualization from data collection, so it will generally be a lot

87 - simpler. Furthermore, if you use `average`, the charts shown in the back-end will match exactly what you see in

88 - Netdata, which is not necessarily true for the other modes of operation.

89 -

90 -5. This code is smart enough, not to slow down Netdata, independently of the speed of the backend server.

91 -

92 -## configuration

93 -

94 -In `/etc/netdata/netdata.conf` you should have something like this (if not download the latest version of `netdata.conf`

95 -from your Netdata):

96 -

97 -```conf

98 -[backend]

99 - enabled = yes | no

101 - host tags = list of TAG=VALUE

102 - destination = space separated list of [PROTOCOL:]HOST[:PORT] - the first working will be used, or a region for kinesis

103 - data source = average | sum | as collected

104 - prefix = Netdata

105 - hostname = my-name

106 - update every = 10

107 - buffer on failures = 10

108 - timeout ms = 20000

109 - send charts matching = *

110 - send hosts matching = localhost *

111 - send names instead of ids = yes

112 -```

113 -

114 -- `enabled = yes | no`, enables or disables sending data to a backend

115 -

117 - type

118 -

119 -- `destination = host1 host2 host3 ...`, accepts **a space separated list** of hostnames, IPs (IPv4 and IPv6) and

120 - ports to connect to. Netdata will use the **first available** to send the metrics.

121 -

122 - The format of each item in this list, is: `[PROTOCOL:]IP[:PORT]`.

123 -

124 - `PROTOCOL` can be `udp` or `tcp`. `tcp` is the default and only supported by the current backends.

125 -

126 - `IP` can be `XX.XX.XX.XX` (IPv4), or `[XX:XX...XX:XX]` (IPv6). For IPv6 you can to enclose the IP in `[]` to

127 - separate it from the port.

128 -

129 - `PORT` can be a number of a service name. If omitted, the default port for the backend will be used

130 - (graphite = 2003, opentsdb = 4242).

131 -

132 - Example IPv4:

133 -

134 -```conf

135 - destination = 10.11.14.2:4242 10.11.14.3:4242 10.11.14.4:4242

136 -```

137 -

138 - Example IPv6 and IPv4 together:

139 -

140 -```conf

141 - destination = [ffff:...:0001]:2003 10.11.12.1:2003

142 -```

143 -

144 - When multiple servers are defined, Netdata will try the next one when the first one fails. This allows you to

145 - load-balance different servers: give your backend servers in different order on each Netdata.

146 -

147 - Netdata also ships `nc-backend.sh`, a script that can be used as a fallback backend to save the

148 - metrics to disk and push them to the time-series database when it becomes available again. It can also be used to

149 - monitor / trace / debug the metrics Netdata generates.

150 -

151 - For kinesis backend `destination` should be set to an AWS region (for example, `us-east-1`).

152 -

153 - The MongoDB backend doesn't use the `destination` option for its configuration. It uses the `mongodb.conf`

154 - [configuration file](/backends/mongodb/README.md) instead.

155 -

156 -- `data source = as collected`, or `data source = average`, or `data source = sum`, selects the kind of data that will

157 - be sent to the backend.

158 -

159 -- `hostname = my-name`, is the hostname to be used for sending data to the backend server. By default this is

160 - `[global].hostname`.

161 -

162 -- `prefix = Netdata`, is the prefix to add to all metrics.

163 -

164 -- `update every = 10`, is the number of seconds between sending data to the backend. Netdata will add some randomness

165 - to this number, to prevent stressing the backend server when many Netdata servers send data to the same backend.

166 - This randomness does not affect the quality of the data, only the time they are sent.

167 -

168 -- `buffer on failures = 10`, is the number of iterations (each iteration is `[backend].update every` seconds) to

169 - buffer data, when the backend is not available. If the backend fails to receive the data after that many failures,

170 - data loss on the backend is expected (Netdata will also log it).

171 -

172 -- `timeout ms = 20000`, is the timeout in milliseconds to wait for the backend server to process the data. By default

173 - this is `2 * update_every * 1000`.

174 -

175 -- `send hosts matching = localhost *` includes one or more space separated patterns, using `*` as wildcard (any number

176 - of times within each pattern). The patterns are checked against the hostname (the localhost is always checked as

177 - `localhost`), allowing us to filter which hosts will be sent to the backend when this Netdata is a central Netdata

178 - aggregating multiple hosts. A pattern starting with `!` gives a negative match. So to match all hosts named `*db*`

179 - except hosts containing `*child*`, use `!*child* *db*` (so, the order is important: the first pattern

180 - matching the hostname will be used - positive or negative).

181 -

182 -- `send charts matching = *` includes one or more space separated patterns, using `*` as wildcard (any number of times

183 - within each pattern). The patterns are checked against both chart id and chart name. A pattern starting with `!`

184 - gives a negative match. So to match all charts named `apps.*` except charts ending in `*reads`, use `!*reads

185 - apps.*` (so, the order is important: the first pattern matching the chart id or the chart name will be used -

186 - positive or negative).

187 -

188 -- `send names instead of ids = yes | no` controls the metric names Netdata should send to backend. Netdata supports

189 - names and IDs for charts and dimensions. Usually IDs are unique identifiers as read by the system and names are

190 - human friendly labels (also unique). Most charts and metrics have the same ID and name, but in several cases they

191 - are different: disks with device-mapper, interrupts, QoS classes, statsd synthetic charts, etc.

192 -

193 -- `host tags = list of TAG=VALUE` defines tags that should be appended on all metrics for the given host. These are

194 - currently only sent to graphite, json, opentsdb and prometheus. Please use the appropriate format for each

195 - time-series db. For example opentsdb likes them like `TAG1=VALUE1 TAG2=VALUE2`, but prometheus like `tag1="value1",

196 - tag2="value2"`. Host tags are mirrored with database replication (streaming of metrics between Netdata servers).

197 -

198 - Starting from Netdata v1.20 the host tags are parsed in accordance with a configured backend type and stored as

199 - host labels so that they can be reused in API responses and exporting connectors. The parsing is supported for

200 - graphite, json, opentsdb, and prometheus (default) backend types. You can check how the host tags were parsed using

201 - the /api/v1/info API call.

202 -

203 -## monitoring operation

204 -

205 -Netdata provides 5 charts:

206 -

207 -1. **Buffered metrics**, the number of metrics Netdata added to the buffer for dispatching them to the

208 - backend server.

209 -

210 -2. **Buffered data size**, the amount of data (in KB) Netdata added the buffer.

211 -

212 -3. ~~**Backend latency**, the time the backend server needed to process the data Netdata sent. If there was a

213 - re-connection involved, this includes the connection time.~~ (this chart has been removed, because it only measures

214 - the time Netdata needs to give the data to the O/S - since the backend servers do not ack the reception, Netdata

215 - does not have any means to measure this properly).

216 -

217 -4. **Backend operations**, the number of operations performed by Netdata.

218 -

219 -5. **Backend thread CPU usage**, the CPU resources consumed by the Netdata thread, that is responsible for sending the

220 - metrics to the backend server.

221 -

222 -![image](https://cloud.githubusercontent.com/assets/2662304/20463536/eb196084-af3d-11e6-8ee5-ddbd3b4d8449.png)

223 -

224 -## alarms

225 -

226 -Netdata adds 4 alarms:

227 -

228 -1. `backend_last_buffering`, number of seconds since the last successful buffering of backend data

229 -2. `backend_metrics_sent`, percentage of metrics sent to the backend server

230 -3. `backend_metrics_lost`, number of metrics lost due to repeating failures to contact the backend server

231 -4. ~~`backend_slow`, the percentage of time between iterations needed by the backend time to process the data sent by

232 - Netdata~~ (this was misleading and has been removed).

233 -

234 -![image](https://cloud.githubusercontent.com/assets/2662304/20463779/a46ed1c2-af43-11e6-91a5-07ca4533cac3.png)

235 -

236 -

backends/TIMESCALE.md deleted

-57

@@ -1,57 +0,0 @@

1 -<!--

2 -title: "Writing metrics to TimescaleDB"

3 -custom_edit_url: https://github.com/netdata/netdata/edit/master/backends/TIMESCALE.md

4 --->

5 -

6 -# Writing metrics to TimescaleDB

7 -

8 -Thanks to Netdata's community of developers and system administrators, and Mahlon Smith

9 -([GitHub](https://github.com/mahlonsmith)/[Website](http://www.martini.nu/)) in particular, Netdata now supports

10 -archiving metrics directly to TimescaleDB.

11 -

12 -What's TimescaleDB? Here's how their team defines the project on their [GitHub page](https://github.com/timescale/timescaledb):

13 -

14 -> TimescaleDB is an open-source database designed to make SQL scalable for time-series data. It is engineered up from

15 -> PostgreSQL, providing automatic partitioning across time and space (partitioning key), as well as full SQL support.

16 -

17 -## Quickstart

18 -

19 -To get started archiving metrics to TimescaleDB right away, check out Mahlon's [`netdata-timescale-relay`

20 -repository](https://github.com/mahlonsmith/netdata-timescale-relay) on GitHub.

21 -

22 -This small program takes JSON streams from a Netdata client and writes them to a PostgreSQL (aka TimescaleDB) table.

23 -You'll run this program in parallel with Netdata, and after a short [configuration

24 -process](https://github.com/mahlonsmith/netdata-timescale-relay#configuration), your metrics should start populating

25 -TimescaleDB.

26 -

27 -Finally, another member of Netdata's community has built a project that quickly launches Netdata, TimescaleDB, and

28 -Grafana in easy-to-manage Docker containers. Rune Juhl Jacobsen's

29 -[project](https://github.com/runejuhl/grafana-timescaledb) uses a `Makefile` to create everything, which makes it

30 -perfect for testing and experimentation.

31 -

32 -## Netdata↔TimescaleDB in action

33 -

34 -Aside from creating incredible contributions to Netdata, Mahlon works at [LAIKA](https://www.laika.com/), an

35 -Oregon-based animation studio that's helped create acclaimed films like _Coraline_ and _Kubo and the Two Strings_.

36 -

37 -As part of his work to maintain the company's infrastructure of render farms, workstations, and virtual machines, he's

38 -using Netdata, `netdata-timescale-relay`, and TimescaleDB to store Netdata metrics alongside other data from other

39 -sources.

40 -

41 -> LAIKA is a long-time PostgreSQL user and added TimescaleDB to their infrastructure in 2018 to help manage and store

42 -> their IT metrics and time-series data. So far, the tool has been in production at LAIKA for over a year and helps them

43 -> with their use case of time-based logging, where they record over 8 million metrics an hour for netdata content alone.

44 -

45 -By archiving Netdata metrics to a backend like TimescaleDB, LAIKA can consolidate metrics data from distributed machines

46 -efficiently. Mahlon can then correlate Netdata metrics with other sources directly in TimescaleDB.

47 -

48 -And, because LAIKA will soon be storing years worth of Netdata metrics data in TimescaleDB, they can analyze long-term

49 -metrics as their films move from concept to final cut.

50 -

51 -Read the full blog post from LAIKA at the [TimescaleDB

52 -blog](https://blog.timescale.com/blog/writing-it-metrics-from-netdata-to-timescaledb/amp/).

53 -

54 -Thank you to Mahlon, Rune, TimescaleDB, and the members of the Netdata community that requested and then built this

55 -backend connection between Netdata and TimescaleDB!

56 -

57 -

backends/WALKTHROUGH.md deleted

-258

@@ -1,258 +0,0 @@

1 -<!--

2 -title: "Netdata, Prometheus, Grafana stack"

3 -custom_edit_url: https://github.com/netdata/netdata/edit/master/backends/WALKTHROUGH.md

4 --->

5 -

6 -# Netdata, Prometheus, Grafana stack

7 -

8 -## Intro

9 -

10 -In this article I will walk you through the basics of getting Netdata, Prometheus and Grafana all working together and

11 -monitoring your application servers. This article will be using docker on your local workstation. We will be working

12 -with docker in an ad-hoc way, launching containers that run ‘/bin/bash’ and attaching a TTY to them. I use docker here

13 -in a purely academic fashion and do not condone running Netdata in a container. I pick this method so individuals

14 -without cloud accounts or access to VMs can try this out and for it’s speed of deployment.

15 -

16 -## Why Netdata, Prometheus, and Grafana

17 -

18 -Some time ago I was introduced to Netdata by a coworker. We were attempting to troubleshoot python code which seemed to

19 -be bottlenecked. I was instantly impressed by the amount of metrics Netdata exposes to you. I quickly added Netdata to

20 -my set of go-to tools when troubleshooting systems performance.

21 -

22 -Some time ago, even later, I was introduced to Prometheus. Prometheus is a monitoring application which flips the normal

23 -architecture around and polls rest endpoints for its metrics. This architectural change greatly simplifies and decreases

24 -the time necessary to begin monitoring your applications. Compared to current monitoring solutions the time spent on

25 -designing the infrastructure is greatly reduced. Running a single Prometheus server per application becomes feasible

26 -with the help of Grafana.

27 -

28 -Grafana has been the go to graphing tool for… some time now. It’s awesome, anyone that has used it knows it’s awesome.

29 -We can point Grafana at Prometheus and use Prometheus as a data source. This allows a pretty simple overall monitoring

30 -architecture: Install Netdata on your application servers, point Prometheus at Netdata, and then point Grafana at

31 -Prometheus.

32 -

33 -I’m omitting an important ingredient in this stack in order to keep this tutorial simple and that is service discovery.

34 -My personal preference is to use Consul. Prometheus can plug into consul and automatically begin to scrape new hosts

35 -that register a Netdata client with Consul.

36 -

37 -At the end of this tutorial you will understand how each technology fits together to create a modern monitoring stack.

38 -This stack will offer you visibility into your application and systems performance.

39 -

40 -## Getting Started - Netdata

41 -

42 -To begin let’s create our container which we will install Netdata on. We need to run a container, forward the necessary

43 -port that Netdata listens on, and attach a tty so we can interact with the bash shell on the container. But before we do

44 -this we want name resolution between the two containers to work. In order to accomplish this we will create a

45 -user-defined network and attach both containers to this network. The first command we should run is:

46 -

47 -```sh

48 -docker network create --driver bridge netdata-tutorial

49 -```

50 -

51 -With this user-defined network created we can now launch our container we will install Netdata on and point it to this

52 -network.

53 -

54 -```sh

55 -docker run -it --name netdata --hostname netdata --network=netdata-tutorial -p 19999:19999 centos:latest '/bin/bash'

56 -```

57 -

58 -This command creates an interactive tty session (-it), gives the container both a name in relation to the docker daemon

59 -and a hostname (this is so you know what container is which when working in the shells and docker maps hostname

60 -resolution to this container), forwards the local port 19999 to the container’s port 19999 (-p 19999:19999), sets the

61 -command to run (/bin/bash) and then chooses the base container images (centos:latest). After running this you should be

62 -sitting inside the shell of the container.

63 -

64 -After we have entered the shell we can install Netdata. This process could not be easier. If you take a look at [this

65 -link](/packaging/installer/README.md), the Netdata devs give us several one-liners to install Netdata. I have not had

66 -any issues with these one liners and their bootstrapping scripts so far (If you guys run into anything do share). Run

67 -the following command in your container.

68 -

69 -```sh

70 -bash <(curl -Ss https://my-netdata.io/kickstart.sh) --dont-wait

71 -```

72 -

73 -After the install completes you should be able to hit the Netdata dashboard at <http://localhost:19999/> (replace

74 -localhost if you’re doing this on a VM or have the docker container hosted on a machine not on your local system). If

75 -this is your first time using Netdata I suggest you take a look around. The amount of time I’ve spent digging through

76 -/proc and calculating my own metrics has been greatly reduced by this tool. Take it all in.

77 -

78 -Next I want to draw your attention to a particular endpoint. Navigate to

79 -<http://localhost:19999/api/v1/allmetrics?format=prometheus&help=yes> In your browser. This is the endpoint which

80 -publishes all the metrics in a format which Prometheus understands. Let’s take a look at one of these metrics.

81 -`netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 0.0831255 1501271696000` This

82 -metric is representing several things which I will go in more details in the section on prometheus. For now understand

83 -that this metric: `netdata_system_cpu_percentage_average` has several labels: (chart, family, dimension). This

84 -corresponds with the first cpu chart you see on the Netdata dashboard.

85 -

86 -![](https://github.com/ldelossa/NetdataTutorial/raw/master/Screen%20Shot%202017-07-28%20at%204.00.45%20PM.png)

87 -

88 -This CHART is called ‘system.cpu’, The FAMILY is cpu, and the DIMENSION we are observing is “system”. You can begin to

89 -draw links between the charts in Netdata to the prometheus metrics format in this manner.

90 -

91 -## Prometheus

92 -

93 -We will be installing prometheus in a container for purpose of demonstration. While prometheus does have an official

94 -container I would like to walk through the install process and setup on a fresh container. This will allow anyone

95 -reading to migrate this tutorial to a VM or Server of any sort.

96 -

97 -Let’s start another container in the same fashion as we did the Netdata container.

98 -

99 -```sh

100 -docker run -it --name prometheus --hostname prometheus

101 ---network=netdata-tutorial -p 9090:9090 centos:latest '/bin/bash'

102 -```

103 -

104 -This should drop you into a shell once again. Once there quickly install your favorite editor as we will be editing

105 -files later in this tutorial.

106 -

107 -```sh

108 -yum install vim -y

109 -```

110 -

111 -Prometheus provides a tarball of their latest stable versions [here](https://prometheus.io/download/).

112 -

113 -Let’s download the latest version and install into your container.

114 -

115 -```sh

116 -cd /tmp && curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest \

117 -| grep "browser_download_url.*linux-amd64.tar.gz" \

118 -| cut -d '"' -f 4 \

119 -| wget -qi -

120 -

121 -mkdir /opt/prometheus

122 -

123 -sudo tar -xvf /tmp/prometheus-*linux-amd64.tar.gz -C /opt/prometheus --strip=1

124 -```

125 -

126 -This should get prometheus installed into the container. Let’s test that we can run prometheus and connect to it’s web

127 -interface.

128 -

129 -```sh

130 -/opt/prometheus/prometheus

131 -```

132 -

133 -Now attempt to go to <http://localhost:9090/>. You should be presented with the prometheus homepage. This is a good

134 -point to talk about Prometheus’s data model which can be viewed here: <https://prometheus.io/docs/concepts/data_model/>

135 -As explained we have two key elements in Prometheus metrics. We have the ‘metric’ and its ‘labels’. Labels allow for

136 -granularity between metrics. Let’s use our previous example to further explain.

137 -

138 -```conf

139 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 0.0831255 1501271696000

140 -```

141 -

142 -Here our metric is ‘netdata_system_cpu_percentage_average’ and our labels are ‘chart’, ‘family’, and ‘dimension. The

143 -last two values constitute the actual metric value for the metric type (gauge, counter, etc…). We can begin graphing

144 -system metrics with this information, but first we need to hook up Prometheus to poll Netdata stats.

145 -

146 -Let’s move our attention to Prometheus’s configuration. Prometheus gets it config from the file located (in our example)

147 -at `/opt/prometheus/prometheus.yml`. I won’t spend an extensive amount of time going over the configuration values

148 -documented here: <https://prometheus.io/docs/operating/configuration/>. We will be adding a new“job” under the

149 -“scrape_configs”. Let’s make the “scrape_configs” section look like this (we can use the dns name Netdata due to the

150 -custom user-defined network we created in docker beforehand).

151 -

152 -```yaml

153 -scrape_configs:

154 - # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.

155 - - job_name: 'prometheus'

156 -

157 - # metrics_path defaults to '/metrics'

158 - # scheme defaults to 'http'.

159 -

160 - static_configs:

161 - - targets: ['localhost:9090']

162 -

163 - - job_name: 'netdata'

164 -

165 - metrics_path: /api/v1/allmetrics

166 - params:

167 - format: [ prometheus ]

168 -

169 - static_configs:

170 - - targets: ['netdata:19999']

171 -```

172 -

173 -Let’s start prometheus once again by running `/opt/prometheus/prometheus`. If we now navigate to prometheus at

174 -‘<http://localhost:9090/targets’> we should see our target being successfully scraped. If we now go back to the

175 -Prometheus’s homepage and begin to type ‘netdata\_’ Prometheus should auto complete metrics it is now scraping.

176 -

177 -![](https://github.com/ldelossa/NetdataTutorial/raw/master/Screen%20Shot%202017-07-28%20at%205.13.43%20PM.png)

178 -

179 -Let’s now start exploring how we can graph some metrics. Back in our Netdata container lets get the CPU spinning with a

180 -pointless busy loop. On the shell do the following:

181 -

182 -```sh

183 -[root@netdata /]# while true; do echo "HOT HOT HOT CPU"; done

184 -```

185 -

186 -Our Netdata cpu graph should be showing some activity. Let’s represent this in Prometheus. In order to do this let’s

187 -keep our metrics page open for reference: <http://localhost:19999/api/v1/allmetrics?format=prometheus&help=yes> We are

188 -setting out to graph the data in the CPU chart so let’s search for “system.cpu”in the metrics page above. We come across

189 -a section of metrics with the first comments `# COMMENT homogeneous chart "system.cpu", context "system.cpu", family

190 -"cpu", units "percentage"` Followed by the metrics. This is a good start now let us drill down to the specific metric we

191 -would like to graph.

192 -

193 -```conf

194 -# COMMENT

195 -netdata_system_cpu_percentage_average: dimension "system", value is percentage, gauge, dt 1501275951 to 1501275951 inclusive

196 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 0.0000000 1501275951000

197 -```

198 -

199 -Here we learn that the metric name we care about is‘netdata_system_cpu_percentage_average’ so throw this into Prometheus

200 -and see what we get. We should see something similar to this (I shut off my busy loop)

201 -

202 -![](https://github.com/ldelossa/NetdataTutorial/raw/master/Screen%20Shot%202017-07-28%20at%205.47.53%20PM.png)

203 -

204 -This is a good step toward what we want. Also make note that Prometheus will tag on an ‘instance’ label for us which

205 -corresponds to our statically defined job in the configuration file. This allows us to tailor our queries to specific

206 -instances. Now we need to isolate the dimension we want in our query. To do this let us refine the query slightly. Let’s

207 -query the dimension also. Place this into our query text box.

208 -`netdata_system_cpu_percentage_average{dimension="system"}` We now wind up with the following graph.

209 -

210 -![](https://github.com/ldelossa/NetdataTutorial/raw/master/Screen%20Shot%202017-07-28%20at%205.54.40%20PM.png)

211 -

212 -Awesome, this is exactly what we wanted. If you haven’t caught on yet we can emulate entire charts from Netdata by using

213 -the `chart` dimension. If you’d like you can combine the ‘chart’ and ‘instance’ dimension to create per-instance charts.

214 -Let’s give this a try: `netdata_system_cpu_percentage_average{chart="system.cpu", instance="netdata:19999"}`

215 -

216 -This is the basics of using Prometheus to query Netdata. I’d advise everyone at this point to read [this

217 -page](/backends/prometheus/README.md#using-netdata-with-prometheus). The key point here is that Netdata can export metrics from

218 -its internal DB or can send metrics “as-collected” by specifying the ‘source=as-collected’ url parameter like so.

219 -<http://localhost:19999/api/v1/allmetrics?format=prometheus&help=yes&types=yes&source=as-collected> If you choose to use

220 -this method you will need to use Prometheus's set of functions here: <https://prometheus.io/docs/querying/functions/> to

221 -obtain useful metrics as you are now dealing with raw counters from the system. For example you will have to use the

222 -`irate()` function over a counter to get that metric's rate per second. If your graphing needs are met by using the

223 -metrics returned by Netdata's internal database (not specifying any source= url parameter) then use that. If you find

224 -limitations then consider re-writing your queries using the raw data and using Prometheus functions to get the desired

225 -chart.

226 -

227 -## Grafana

228 -

229 -Finally we make it to grafana. This is the easiest part in my opinion. This time we will actually run the official

230 -grafana docker container as all configuration we need to do is done via the GUI. Let’s run the following command:

231 -

232 -```sh

233 -docker run -i -p 3000:3000 --network=netdata-tutorial grafana/grafana

234 -```

235 -

236 -This will get grafana running at ‘<http://localhost:3000/’> Let’s go there and

237 -

238 -login using the credentials Admin:Admin.

239 -

240 -The first thing we want to do is click ‘Add data source’. Let’s make it look like the following screenshot

241 -

242 -![](https://github.com/ldelossa/NetdataTutorial/raw/master/Screen%20Shot%202017-07-28%20at%206.36.55%20PM.png)

243 -

244 -With this completed let’s graph! Create a new Dashboard by clicking on the top left Grafana Icon and create a new graph

245 -in that dashboard. Fill in the query like we did above and save.

246 -

247 -![](https://github.com/ldelossa/NetdataTutorial/raw/master/Screen%20Shot%202017-07-28%20at%206.39.38%20PM.png)

248 -

249 -## Conclusion

250 -

251 -There you have it, a complete systems monitoring stack which is very easy to deploy. From here I would begin to

252 -understand how Prometheus and a service discovery mechanism such as Consul can play together nicely. My current prod

253 -deployments automatically register Netdata services into Consul and Prometheus automatically begins to scrape them. Once

254 -achieved you do not have to think about the monitoring system until Prometheus cannot keep up with your scale. Once this

255 -happens there are options presented in the Prometheus documentation for solving this. Hope this was helpful, happy

256 -monitoring.

257 -

258 -

backends/aws_kinesis/Makefile.am deleted

-12

@@ -1,12 +0,0 @@

1 -# SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -AUTOMAKE_OPTIONS = subdir-objects

4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in

5 -

6 -dist_noinst_DATA = \

7 - README.md \

8 - $(NULL)

9 -

10 -dist_libconfig_DATA = \

11 - aws_kinesis.conf \

12 - $(NULL)

backends/aws_kinesis/README.md deleted

-53

@@ -1,53 +0,0 @@

1 -<!--

2 -title: "Using Netdata with AWS Kinesis Data Streams"

3 -custom_edit_url: https://github.com/netdata/netdata/edit/master/backends/aws_kinesis/README.md

4 --->

5 -

6 -# Using Netdata with AWS Kinesis Data Streams

7 -

8 -## Prerequisites

9 -

10 -To use AWS Kinesis as a backend AWS SDK for C++ should be

11 -[installed](https://docs.aws.amazon.com/en_us/sdk-for-cpp/v1/developer-guide/setup.html) first. `libcrypto`, `libssl`,

12 -and `libcurl` are also required to compile Netdata with Kinesis support enabled. Next, Netdata should be re-installed

13 -from the source. The installer will detect that the required libraries are now available.

14 -

15 -If the AWS SDK for C++ is being installed from source, it is useful to set `-DBUILD_ONLY="kinesis"`. Otherwise, the

16 -building process could take a very long time. Take a note, that the default installation path for the libraries is

17 -`/usr/local/lib64`. Many Linux distributions don't include this path as the default one for a library search, so it is

18 -advisable to use the following options to `cmake` while building the AWS SDK:

19 -

20 -```sh

21 -cmake -DCMAKE_INSTALL_LIBDIR=/usr/lib -DCMAKE_INSTALL_INCLUDEDIR=/usr/include -DBUILD_SHARED_LIBS=OFF -DBUILD_ONLY=kinesis <aws-sdk-cpp sources>

22 -```

23 -

24 -## Configuration

25 -

26 -To enable data sending to the kinesis backend set the following options in `netdata.conf`:

27 -

28 -```conf

29 -[backend]

30 - enabled = yes

31 - type = kinesis

32 - destination = us-east-1

33 -```

34 -

35 -set the `destination` option to an AWS region.

36 -

37 -In the Netdata configuration directory run `./edit-config aws_kinesis.conf` and set AWS credentials and stream name:

38 -

39 -```yaml

40 -# AWS credentials

41 -aws_access_key_id = your_access_key_id

42 -aws_secret_access_key = your_secret_access_key

43 -

44 -# destination stream

45 -stream name = your_stream_name

46 -```

47 -

48 -Alternatively, AWS credentials can be set for the `netdata` user using AWS SDK for C++ [standard methods](https://docs.aws.amazon.com/sdk-for-cpp/v1/developer-guide/credentials.html).

49 -

50 -A partition key for every record is computed automatically by Netdata with the purpose to distribute records across

51 -available shards evenly.

52 -

53 -

backends/aws_kinesis/aws_kinesis.c deleted

-94

@@ -1,94 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#define BACKENDS_INTERNALS

4 -#include "aws_kinesis.h"

5 -

6 -#define CONFIG_FILE_LINE_MAX ((CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 1024) * 2)

7 -

8 -// ----------------------------------------------------------------------------

9 -// kinesis backend

10 -

11 -// read the aws_kinesis.conf file

12 -int read_kinesis_conf(const char *path, char **access_key_id_p, char **secret_access_key_p, char **stream_name_p)

13 -{

14 - char *access_key_id = *access_key_id_p;

15 - char *secret_access_key = *secret_access_key_p;

16 - char *stream_name = *stream_name_p;

17 -

18 - if(unlikely(access_key_id)) freez(access_key_id);

19 - if(unlikely(secret_access_key)) freez(secret_access_key);

20 - if(unlikely(stream_name)) freez(stream_name);

21 - access_key_id = NULL;

22 - secret_access_key = NULL;

23 - stream_name = NULL;

24 -

25 - int line = 0;

26 -

27 - char filename[FILENAME_MAX + 1];

28 - snprintfz(filename, FILENAME_MAX, "%s/aws_kinesis.conf", path);

29 -

30 - char buffer[CONFIG_FILE_LINE_MAX + 1], *s;

31 -

32 - debug(D_BACKEND, "BACKEND: opening config file '%s'", filename);

33 -

34 - FILE *fp = fopen(filename, "r");

35 - if(!fp) {

36 - return 1;

37 - }

38 -

39 - while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) {

40 - buffer[CONFIG_FILE_LINE_MAX] = '\0';

41 - line++;

42 -

43 - s = trim(buffer);

44 - if(!s || *s == '#') {

45 - debug(D_BACKEND, "BACKEND: ignoring line %d of file '%s', it is empty.", line, filename);

46 - continue;

47 - }

48 -

49 - char *name = s;

50 - char *value = strchr(s, '=');

51 - if(unlikely(!value)) {

52 - error("BACKEND: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename);

53 - continue;

54 - }

55 - *value = '\0';

56 - value++;

57 -

58 - name = trim(name);

59 - value = trim(value);

60 -

61 - if(unlikely(!name || *name == '#')) {

62 - error("BACKEND: ignoring line %d of file '%s', name is empty.", line, filename);

63 - continue;

64 - }

65 -

66 - if(!value)

67 - value = "";

68 - else

69 - value = strip_quotes(value);

70 -

71 - if(name[0] == 'a' && name[4] == 'a' && !strcmp(name, "aws_access_key_id")) {

72 - access_key_id = strdupz(value);

73 - }

74 - else if(name[0] == 'a' && name[4] == 's' && !strcmp(name, "aws_secret_access_key")) {

75 - secret_access_key = strdupz(value);

76 - }

77 - else if(name[0] == 's' && !strcmp(name, "stream name")) {

78 - stream_name = strdupz(value);

79 - }

80 - }

81 -

82 - fclose(fp);

83 -

84 - if(unlikely(!stream_name || !*stream_name)) {

85 - error("BACKEND: stream name is a mandatory Kinesis parameter but it is not configured");

86 - return 1;

87 - }

88 -

89 - *access_key_id_p = access_key_id;

90 - *secret_access_key_p = secret_access_key;

91 - *stream_name_p = stream_name;

92 -

93 - return 0;

94 -}

backends/aws_kinesis/aws_kinesis.conf deleted

-10

@@ -1,10 +0,0 @@

1 -# AWS Kinesis Data Streams backend configuration

2 -#

3 -# All options in this file are mandatory

4 -

5 -# AWS credentials

6 -aws_access_key_id =

7 -aws_secret_access_key =

8 -

9 -# destination stream

10 -stream name =

\ No newline at end of file

backends/aws_kinesis/aws_kinesis.h deleted

-14

@@ -1,14 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_BACKEND_KINESIS_H

4 -#define NETDATA_BACKEND_KINESIS_H

5 -

6 -#include "backends/backends.h"

7 -#include "aws_kinesis_put_record.h"

8 -

9 -#define KINESIS_PARTITION_KEY_MAX 256

10 -#define KINESIS_RECORD_MAX 1024 * 1024

11 -

12 -extern int read_kinesis_conf(const char *path, char **auth_key_id_p, char **secure_key_p, char **stream_name_p);

13 -

14 -#endif //NETDATA_BACKEND_KINESIS_H

backends/aws_kinesis/aws_kinesis_put_record.cc deleted

-87

@@ -1,87 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#include <aws/core/Aws.h>

4 -#include <aws/core/client/ClientConfiguration.h>

5 -#include <aws/core/auth/AWSCredentials.h>

6 -#include <aws/core/utils/Outcome.h>

7 -#include <aws/kinesis/KinesisClient.h>

8 -#include <aws/kinesis/model/PutRecordRequest.h>

9 -#include "aws_kinesis_put_record.h"

10 -

11 -using namespace Aws;

12 -

13 -static SDKOptions options;

14 -

15 -static Kinesis::KinesisClient *client;

16 -

17 -struct request_outcome {

18 - Kinesis::Model::PutRecordOutcomeCallable future_outcome;

19 - size_t data_len;

20 -};

21 -

22 -static Vector<request_outcome> request_outcomes;

23 -

24 -void backends_kinesis_init(const char *region, const char *access_key_id, const char *secret_key, const long timeout) {

25 - InitAPI(options);

26 -

27 - Client::ClientConfiguration config;

28 -

29 - config.region = region;

30 - config.requestTimeoutMs = timeout;

31 - config.connectTimeoutMs = timeout;

32 -

33 - if(access_key_id && *access_key_id && secret_key && *secret_key) {

34 - client = New<Kinesis::KinesisClient>("client", Auth::AWSCredentials(access_key_id, secret_key), config);

35 - } else {

36 - client = New<Kinesis::KinesisClient>("client", config);

37 - }

38 -}

39 -

40 -void backends_kinesis_shutdown() {

41 - Delete(client);

42 -

43 - ShutdownAPI(options);

44 -}

45 -

46 -int backends_kinesis_put_record(const char *stream_name, const char *partition_key,

47 - const char *data, size_t data_len) {

48 - Kinesis::Model::PutRecordRequest request;

49 -

50 - request.SetStreamName(stream_name);

51 - request.SetPartitionKey(partition_key);

52 - request.SetData(Utils::ByteBuffer((unsigned char*) data, data_len));

53 -

54 - request_outcomes.push_back({client->PutRecordCallable(request), data_len});

55 -

56 - return 0;

57 -}

58 -

59 -int backends_kinesis_get_result(char *error_message, size_t *sent_bytes, size_t *lost_bytes) {

60 - Kinesis::Model::PutRecordOutcome outcome;

61 - *sent_bytes = 0;

62 - *lost_bytes = 0;

63 -

64 - for(auto request_outcome = request_outcomes.begin(); request_outcome != request_outcomes.end(); ) {

65 - std::future_status status = request_outcome->future_outcome.wait_for(std::chrono::microseconds(100));

66 -

67 - if(status == std::future_status::ready || status == std::future_status::deferred) {

68 - outcome = request_outcome->future_outcome.get();

69 - *sent_bytes += request_outcome->data_len;

70 -

71 - if(!outcome.IsSuccess()) {

72 - *lost_bytes += request_outcome->data_len;

73 - outcome.GetError().GetMessage().copy(error_message, ERROR_LINE_MAX);

74 - }

75 -

76 - request_outcomes.erase(request_outcome);

77 - } else {

78 - ++request_outcome;

79 - }

80 - }

81 -

82 - if(*lost_bytes) {

83 - return 1;

84 - }

85 -

86 - return 0;

87 -}

\ No newline at end of file

backends/aws_kinesis/aws_kinesis_put_record.h deleted

-25

@@ -1,25 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_BACKEND_KINESIS_PUT_RECORD_H

4 -#define NETDATA_BACKEND_KINESIS_PUT_RECORD_H

5 -

6 -#define ERROR_LINE_MAX 1023

7 -

8 -#ifdef __cplusplus

9 -extern "C" {

10 -#endif

11 -

12 -void backends_kinesis_init(const char *region, const char *access_key_id, const char *secret_key, const long timeout);

13 -

14 -void backends_kinesis_shutdown();

15 -

16 -int backends_kinesis_put_record(const char *stream_name, const char *partition_key,

17 - const char *data, size_t data_len);

18 -

19 -int backends_kinesis_get_result(char *error_message, size_t *sent_bytes, size_t *lost_bytes);

20 -

21 -#ifdef __cplusplus

22 -}

23 -#endif

24 -

25 -#endif //NETDATA_BACKEND_KINESIS_PUT_RECORD_H

backends/backends.c deleted

-1251

@@ -1,1251 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#define BACKENDS_INTERNALS

4 -#include "backends.h"

5 -

6 -// ----------------------------------------------------------------------------

7 -// How backends work in netdata:

8 -//

9 -// 1. There is an independent thread that runs at the required interval

10 -// (for example, once every 10 seconds)

11 -//

12 -// 2. Every time it wakes, it calls the backend formatting functions to build

13 -// a buffer of data. This is a very fast, memory only operation.

14 -//

15 -// 3. If the buffer already includes data, the new data are appended.

16 -// If the buffer becomes too big, because the data cannot be sent, a

17 -// log is written and the buffer is discarded.

18 -//

19 -// 4. Then it tries to send all the data. It blocks until all the data are sent

20 -// or the socket returns an error.

21 -// If the time required for this is above the interval, it starts skipping

22 -// intervals, but the calculated values include the entire database, without

23 -// gaps (it remembers the timestamps and continues from where it stopped).

24 -//

25 -// 5. repeats the above forever.

26 -//

27 -

28 -const char *global_backend_prefix = "netdata";

29 -const char *global_backend_send_charts_matching = "*";

30 -int global_backend_update_every = 10;

31 -BACKEND_OPTIONS global_backend_options = BACKEND_SOURCE_DATA_AVERAGE | BACKEND_OPTION_SEND_NAMES;

32 -const char *global_backend_source = NULL;

33 -

34 -// ----------------------------------------------------------------------------

35 -// helper functions for backends

36 -

37 -size_t backend_name_copy(char *d, const char *s, size_t usable) {

38 - size_t n;

39 -

40 - for(n = 0; *s && n < usable ; d++, s++, n++) {

41 - char c = *s;

42 -

43 - if(c != '.' && !isalnum(c)) *d = '_';

44 - else *d = c;

45 - }

46 - *d = '\0';

47 -

48 - return n;

49 -}

50 -

51 -// calculate the SUM or AVERAGE of a dimension, for any timeframe

52 -// may return NAN if the database does not have any value in the give timeframe

53 -

54 -calculated_number backend_calculate_value_from_stored_data(

55 - RRDSET *st // the chart

56 - , RRDDIM *rd // the dimension

57 - , time_t after // the start timestamp

58 - , time_t before // the end timestamp

59 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

60 - , time_t *first_timestamp // the first point of the database used in this response

61 - , time_t *last_timestamp // the timestamp that should be reported to backend

62 -) {

63 - RRDHOST *host = st->rrdhost;

64 - (void)host;

65 -

66 - // find the edges of the rrd database for this chart

67 - time_t first_t = rd->state->query_ops.oldest_time(rd);

68 - time_t last_t = rd->state->query_ops.latest_time(rd);

69 - time_t update_every = st->update_every;

70 - struct rrddim_query_handle handle;

71 - storage_number n;

72 -

73 - // step back a little, to make sure we have complete data collection

74 - // for all metrics

75 - after -= update_every * 2;

76 - before -= update_every * 2;

77 -

78 - // align the time-frame

79 - after = after - (after % update_every);

80 - before = before - (before % update_every);

81 -

82 - // for before, loose another iteration

83 - // the latest point will be reported the next time

84 - before -= update_every;

85 -

86 - if(unlikely(after > before))

87 - // this can happen when update_every > before - after

88 - after = before;

89 -

90 - if(unlikely(after < first_t))

91 - after = first_t;

92 -

93 - if(unlikely(before > last_t))

94 - before = last_t;

95 -

96 - if(unlikely(before < first_t || after > last_t)) {

97 - // the chart has not been updated in the wanted timeframe

98 - debug(D_BACKEND, "BACKEND: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu",

99 - host->hostname, st->id, rd->id,

100 - (unsigned long)after, (unsigned long)before,

101 - (unsigned long)first_t, (unsigned long)last_t

102 - );

103 - return NAN;

104 - }

105 -

106 - *first_timestamp = after;

107 - *last_timestamp = before;

108 -

109 - size_t counter = 0;

110 - calculated_number sum = 0;

111 -

112 -/*

113 - long start_at_slot = rrdset_time2slot(st, before),

114 - stop_at_slot = rrdset_time2slot(st, after),

115 - slot, stop_now = 0;

116 -

117 - for(slot = start_at_slot; !stop_now ; slot--) {

118 -

119 - if(unlikely(slot < 0)) slot = st->entries - 1;

120 - if(unlikely(slot == stop_at_slot)) stop_now = 1;

121 -

122 - storage_number n = rd->values[slot];

123 -

124 - if(unlikely(!does_storage_number_exist(n))) {

125 - // not collected

126 - continue;

127 - }

128 -

129 - calculated_number value = unpack_storage_number(n);

130 - sum += value;

131 -

132 - counter++;

133 - }

134 -*/

135 - for(rd->state->query_ops.init(rd, &handle, after, before) ; !rd->state->query_ops.is_finished(&handle) ; ) {

136 - time_t curr_t;

137 - n = rd->state->query_ops.next_metric(&handle, &curr_t);

138 -

139 - if(unlikely(!does_storage_number_exist(n))) {

140 - // not collected

141 - continue;

142 - }

143 -

144 - calculated_number value = unpack_storage_number(n);

145 - sum += value;

146 -

147 - counter++;

148 - }

149 - rd->state->query_ops.finalize(&handle);

150 - if(unlikely(!counter)) {

151 - debug(D_BACKEND, "BACKEND: %s.%s.%s: no values stored in database for range %lu to %lu",

152 - host->hostname, st->id, rd->id,

153 - (unsigned long)after, (unsigned long)before

154 - );

155 - return NAN;

156 - }

157 -

158 - if(unlikely(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM))

159 - return sum;

160 -

161 - return sum / (calculated_number)counter;

162 -}

163 -

164 -

165 -// discard a response received by a backend

166 -// after logging a simple of it to error.log

167 -

168 -int discard_response(BUFFER *b, const char *backend) {

169 - char sample[1024];

170 - const char *s = buffer_tostring(b);

171 - char *d = sample, *e = &sample[sizeof(sample) - 1];

172 -

173 - for(; *s && d < e ;s++) {

174 - char c = *s;

175 - if(unlikely(!isprint(c))) c = ' ';

176 - *d++ = c;

177 - }

178 - *d = '\0';

179 -

180 - info("BACKEND: received %zu bytes from %s backend. Ignoring them. Sample: '%s'", buffer_strlen(b), backend, sample);

181 - buffer_flush(b);

182 - return 0;

183 -}

184 -

185 -

186 -// ----------------------------------------------------------------------------

187 -// the backend thread

188 -

189 -static SIMPLE_PATTERN *charts_pattern = NULL;

190 -static SIMPLE_PATTERN *hosts_pattern = NULL;

191 -

192 -inline int backends_can_send_rrdset(BACKEND_OPTIONS backend_options, RRDSET *st) {

193 - RRDHOST *host = st->rrdhost;

194 - (void)host;

195 -

196 - // Do not send anomaly rates charts.

197 - if (unlikely(st->state->is_ar_chart))

198 - return 0;

199 -

200 - if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_BACKEND_IGNORE)))

201 - return 0;

202 -

203 - if(unlikely(!rrdset_flag_check(st, RRDSET_FLAG_BACKEND_SEND))) {

204 - // we have not checked this chart

205 - if(simple_pattern_matches(charts_pattern, st->id) || simple_pattern_matches(charts_pattern, st->name))

206 - rrdset_flag_set(st, RRDSET_FLAG_BACKEND_SEND);

207 - else {

208 - rrdset_flag_set(st, RRDSET_FLAG_BACKEND_IGNORE);

209 - debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.", st->id, host->hostname);

210 - return 0;

211 - }

212 - }

213 -

214 - if(unlikely(!rrdset_is_available_for_exporting_and_alarms(st))) {

215 - debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is not available for backends.", st->id, host->hostname);

216 - return 0;

217 - }

218 -

219 - if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED))) {

220 - debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.", st->id, host->hostname, rrd_memory_mode_name(host->rrd_memory_mode));

221 - return 0;

222 - }

223 -

224 - return 1;

225 -}

226 -

227 -inline BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options) {

228 - if(!strcmp(source, "raw") || !strcmp(source, "as collected") || !strcmp(source, "as-collected") || !strcmp(source, "as_collected") || !strcmp(source, "ascollected")) {

229 - backend_options |= BACKEND_SOURCE_DATA_AS_COLLECTED;

230 - backend_options &= ~(BACKEND_OPTIONS_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AS_COLLECTED);

231 - }

232 - else if(!strcmp(source, "average")) {

233 - backend_options |= BACKEND_SOURCE_DATA_AVERAGE;

234 - backend_options &= ~(BACKEND_OPTIONS_SOURCE_BITS ^ BACKEND_SOURCE_DATA_AVERAGE);

235 - }

236 - else if(!strcmp(source, "sum") || !strcmp(source, "volume")) {

237 - backend_options |= BACKEND_SOURCE_DATA_SUM;

238 - backend_options &= ~(BACKEND_OPTIONS_SOURCE_BITS ^ BACKEND_SOURCE_DATA_SUM);

239 - }

240 - else {

241 - error("BACKEND: invalid data source method '%s'.", source);

242 - }

243 -

244 - return backend_options;

245 -}

246 -

247 -static void backends_main_cleanup(void *ptr) {

248 - struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;

249 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;

250 -

251 - info("cleaning up...");

252 -

253 - static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

254 -}

255 -

256 -/**

257 - * Set Kinesis variables

258 - *

259 - * Set the variables necessary to work with this specific backend.

260 - *

261 - * @param default_port the default port of the backend

262 - * @param brc function called to check the result.

263 - * @param brf function called to format the message to the backend

264 - */

265 -void backend_set_kinesis_variables(int *default_port,

266 - backend_response_checker_t brc,

267 - backend_request_formatter_t brf)

268 -{

269 - (void)default_port;

270 -#ifndef HAVE_KINESIS

271 - (void)brc;

272 - (void)brf;

273 -#endif

274 -

275 -#if HAVE_KINESIS

276 - *brc = process_json_response;

277 - if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)

278 - *brf = backends_format_dimension_collected_json_plaintext;

279 - else

280 - *brf = backends_format_dimension_stored_json_plaintext;

281 -#endif

282 -}

283 -

284 -/**

285 - * Set Prometheus variables

286 - *

287 - * Set the variables necessary to work with this specific backend.

288 - *

289 - * @param default_port the default port of the backend

290 - * @param brc function called to check the result.

291 - * @param brf function called to format the message to the backend

292 - */

293 -void backend_set_prometheus_variables(int *default_port,

294 - backend_response_checker_t brc,

295 - backend_request_formatter_t brf)

296 -{

297 - (void)default_port;

298 - (void)brf;

299 -#ifndef ENABLE_PROMETHEUS_REMOTE_WRITE

300 - (void)brc;

301 -#endif

302 -

303 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

304 - *brc = backends_process_prometheus_remote_write_response;

305 -#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */

306 -}

307 -

308 -/**

309 - * Set MongoDB variables

310 - *

311 - * Set the variables necessary to work with this specific backend.

312 - *

313 - * @param default_port the default port of the backend

314 - * @param brc function called to check the result.

315 - * @param brf function called to format the message to the backend

316 - */

317 -void backend_set_mongodb_variables(int *default_port,

318 - backend_response_checker_t brc,

319 - backend_request_formatter_t brf)

320 -{

321 - (void)default_port;

322 -#ifndef HAVE_MONGOC

323 - (void)brc;

324 - (void)brf;

325 -#endif

326 -

327 -#if HAVE_MONGOC

328 - *brc = process_json_response;

329 - if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)

330 - *brf = backends_format_dimension_collected_json_plaintext;

331 - else

332 - *brf = backends_format_dimension_stored_json_plaintext;

333 -#endif

334 -}

335 -

336 -/**

337 - * Set JSON variables

338 - *

339 - * Set the variables necessary to work with this specific backend.

340 - *

341 - * @param default_port the default port of the backend

342 - * @param brc function called to check the result.

343 - * @param brf function called to format the message to the backend

344 - */

345 -void backend_set_json_variables(int *default_port,

346 - backend_response_checker_t brc,

347 - backend_request_formatter_t brf)

348 -{

349 - *default_port = 5448;

350 - *brc = process_json_response;

351 -

352 - if (BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)

353 - *brf = backends_format_dimension_collected_json_plaintext;

354 - else

355 - *brf = backends_format_dimension_stored_json_plaintext;

356 -}

357 -

358 -/**

359 - * Set OpenTSDB HTTP variables

360 - *

361 - * Set the variables necessary to work with this specific backend.

362 - *

363 - * @param default_port the default port of the backend

364 - * @param brc function called to check the result.

365 - * @param brf function called to format the message to the backend

366 - */

367 -void backend_set_opentsdb_http_variables(int *default_port,

368 - backend_response_checker_t brc,

369 - backend_request_formatter_t brf)

370 -{

371 - *default_port = 4242;

372 - *brc = process_opentsdb_response;

373 -

374 - if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)

375 - *brf = backends_format_dimension_collected_opentsdb_http;

376 - else

377 - *brf = backends_format_dimension_stored_opentsdb_http;

378 -

379 -}

380 -

381 -/**

382 - * Set OpenTSDB Telnet variables

383 - *

384 - * Set the variables necessary to work with this specific backend.

385 - *

386 - * @param default_port the default port of the backend

387 - * @param brc function called to check the result.

388 - * @param brf function called to format the message to the backend

389 - */

390 -void backend_set_opentsdb_telnet_variables(int *default_port,

391 - backend_response_checker_t brc,

392 - backend_request_formatter_t brf)

393 -{

394 - *default_port = 4242;

395 - *brc = process_opentsdb_response;

396 -

397 - if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)

398 - *brf = backends_format_dimension_collected_opentsdb_telnet;

399 - else

400 - *brf = backends_format_dimension_stored_opentsdb_telnet;

401 -}

402 -

403 -/**

404 - * Set Graphite variables

405 - *

406 - * Set the variables necessary to work with this specific backend.

407 - *

408 - * @param default_port the default port of the backend

409 - * @param brc function called to check the result.

410 - * @param brf function called to format the message to the backend

411 - */

412 -void backend_set_graphite_variables(int *default_port,

413 - backend_response_checker_t brc,

414 - backend_request_formatter_t brf)

415 -{

416 - *default_port = 2003;

417 - *brc = process_graphite_response;

418 -

419 - if(BACKEND_OPTIONS_DATA_SOURCE(global_backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)

420 - *brf = backends_format_dimension_collected_graphite_plaintext;

421 - else

422 - *brf = backends_format_dimension_stored_graphite_plaintext;

423 -}

424 -

425 -/**

426 - * Select Type

427 - *

428 - * Select the backend type based in the user input

429 - *

430 - * @param type is the string that defines the backend type

431 - *

432 - * @return It returns the backend id.

433 - */

434 -BACKEND_TYPE backend_select_type(const char *type) {

435 - if(!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {

436 - return BACKEND_TYPE_GRAPHITE;

437 - }

438 - else if(!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {

439 - return BACKEND_TYPE_OPENTSDB_USING_TELNET;

440 - }

441 - else if(!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) {

442 - return BACKEND_TYPE_OPENTSDB_USING_HTTP;

443 - }

444 - else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {

445 - return BACKEND_TYPE_JSON;

446 - }

447 - else if (!strcmp(type, "prometheus_remote_write")) {

448 - return BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE;

449 - }

450 - else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {

451 - return BACKEND_TYPE_KINESIS;

452 - }

453 - else if (!strcmp(type, "mongodb") || !strcmp(type, "mongodb:plaintext")) {

454 - return BACKEND_TYPE_MONGODB;

455 - }

456 -

457 - return BACKEND_TYPE_UNKNOWN;

458 -}

459 -

460 -/**

461 - * Backend main

462 - *

463 - * The main thread used to control the backends.

464 - *

465 - * @param ptr a pointer to netdata_static_structure.

466 - *

467 - * @return It always return NULL.

468 - */

469 -void *backends_main(void *ptr) {

470 - netdata_thread_cleanup_push(backends_main_cleanup, ptr);

471 -

472 - int default_port = 0;

473 - int sock = -1;

474 - BUFFER *b = buffer_create(1), *response = buffer_create(1);

475 - int (*backend_request_formatter)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS) = NULL;

476 - int (*backend_response_checker)(BUFFER *) = NULL;

477 -

478 -#if HAVE_KINESIS

479 - int do_kinesis = 0;

480 - char *kinesis_auth_key_id = NULL, *kinesis_secure_key = NULL, *kinesis_stream_name = NULL;

481 -#endif

482 -

483 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

484 - int do_prometheus_remote_write = 0;

485 - BUFFER *http_request_header = NULL;

486 -#endif

487 -

488 -#if HAVE_MONGOC

489 - int do_mongodb = 0;

490 - char *mongodb_uri = NULL;

491 - char *mongodb_database = NULL;

492 - char *mongodb_collection = NULL;

493 -

494 - // set the default socket timeout in ms

495 - int32_t mongodb_default_socket_timeout = (int32_t)(global_backend_update_every >= 2)?(global_backend_update_every * MSEC_PER_SEC - 500):1000;

496 -

497 -#endif

498 -

499 -#ifdef ENABLE_HTTPS

500 - struct netdata_ssl opentsdb_ssl = {NULL , NETDATA_SSL_START};

501 -#endif

502 -

503 - // ------------------------------------------------------------------------

504 - // collect configuration options

505 -

506 - struct timeval timeout = {

507 - .tv_sec = 0,

508 - .tv_usec = 0

509 - };

510 - int enabled = config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", 0);

511 - const char *source = config_get(CONFIG_SECTION_BACKEND, "data source", "average");

512 - const char *type = config_get(CONFIG_SECTION_BACKEND, "type", "graphite");

513 - const char *destination = config_get(CONFIG_SECTION_BACKEND, "destination", "localhost");

514 - global_backend_prefix = config_get(CONFIG_SECTION_BACKEND, "prefix", "netdata");

515 - const char *hostname = config_get(CONFIG_SECTION_BACKEND, "hostname", localhost->hostname);

516 - global_backend_update_every = (int)config_get_number(CONFIG_SECTION_BACKEND, "update every", global_backend_update_every);

517 - int buffer_on_failures = (int)config_get_number(CONFIG_SECTION_BACKEND, "buffer on failures", 10);

518 - long timeoutms = config_get_number(CONFIG_SECTION_BACKEND, "timeout ms", global_backend_update_every * 2 * 1000);

519 -

520 - if(config_get_boolean(CONFIG_SECTION_BACKEND, "send names instead of ids", (global_backend_options & BACKEND_OPTION_SEND_NAMES)))

521 - global_backend_options |= BACKEND_OPTION_SEND_NAMES;

522 - else

523 - global_backend_options &= ~BACKEND_OPTION_SEND_NAMES;

524 -

525 - charts_pattern = simple_pattern_create(

526 - global_backend_send_charts_matching = config_get(CONFIG_SECTION_BACKEND, "send charts matching", "*"),

527 - NULL,

528 - SIMPLE_PATTERN_EXACT);

529 - hosts_pattern = simple_pattern_create(config_get(CONFIG_SECTION_BACKEND, "send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);

530 -

531 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

532 - const char *remote_write_path = config_get(CONFIG_SECTION_BACKEND, "remote write URL path", "/receive");

533 -#endif

534 -

535 - // ------------------------------------------------------------------------

536 - // validate configuration options

537 - // and prepare for sending data to our backend

538 -

539 - global_backend_options = backend_parse_data_source(source, global_backend_options);

540 - global_backend_source = source;

541 -

542 - if(timeoutms < 1) {

543 - error("BACKEND: invalid timeout %ld ms given. Assuming %d ms.", timeoutms, global_backend_update_every * 2 * 1000);

544 - timeoutms = global_backend_update_every * 2 * 1000;

545 - }

546 - timeout.tv_sec = (timeoutms * 1000) / 1000000;

547 - timeout.tv_usec = (timeoutms * 1000) % 1000000;

548 -

549 - if(!enabled || global_backend_update_every < 1)

550 - goto cleanup;

551 -

552 - // ------------------------------------------------------------------------

553 - // select the backend type

554 - BACKEND_TYPE work_type = backend_select_type(type);

555 - if (work_type == BACKEND_TYPE_UNKNOWN) {

556 - error("BACKEND: Unknown backend type '%s'", type);

557 - goto cleanup;

558 - }

559 -

560 - switch (work_type) {

561 - case BACKEND_TYPE_OPENTSDB_USING_HTTP: {

562 -#ifdef ENABLE_HTTPS

563 - if (!strcmp(type, "opentsdb:https")) {

564 - security_start_ssl(NETDATA_SSL_CONTEXT_EXPORTING);

565 - }

566 -#endif

567 - backend_set_opentsdb_http_variables(&default_port,&backend_response_checker,&backend_request_formatter);

568 - break;

569 - }

570 - case BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE: {

571 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

572 - do_prometheus_remote_write = 1;

573 -

574 - http_request_header = buffer_create(1);

575 - backends_init_write_request();

576 -#else

577 - error("BACKEND: Prometheus remote write support isn't compiled");

578 -#endif // ENABLE_PROMETHEUS_REMOTE_WRITE

579 - backend_set_prometheus_variables(&default_port,&backend_response_checker,&backend_request_formatter);

580 - break;

581 - }

582 - case BACKEND_TYPE_KINESIS: {

583 -#if HAVE_KINESIS

584 - do_kinesis = 1;

585 -

586 - if(unlikely(read_kinesis_conf(netdata_configured_user_config_dir, &kinesis_auth_key_id, &kinesis_secure_key, &kinesis_stream_name))) {

587 - error("BACKEND: kinesis backend type is set but cannot read its configuration from %s/aws_kinesis.conf", netdata_configured_user_config_dir);

588 - goto cleanup;

589 - }

590 -

591 - backends_kinesis_init(destination, kinesis_auth_key_id, kinesis_secure_key, timeout.tv_sec * 1000 + timeout.tv_usec / 1000);

592 -#else

593 - error("BACKEND: AWS Kinesis support isn't compiled");

594 -#endif // HAVE_KINESIS

595 - backend_set_kinesis_variables(&default_port,&backend_response_checker,&backend_request_formatter);

596 - break;

597 - }

598 - case BACKEND_TYPE_MONGODB: {

599 -#if HAVE_MONGOC

600 - if(unlikely(read_mongodb_conf(netdata_configured_user_config_dir,

601 - &mongodb_uri,

602 - &mongodb_database,

603 - &mongodb_collection))) {

604 - error("BACKEND: mongodb backend type is set but cannot read its configuration from %s/mongodb.conf",

605 - netdata_configured_user_config_dir);

606 - goto cleanup;

607 - }

608 -

609 - if(likely(!backends_mongodb_init(mongodb_uri, mongodb_database, mongodb_collection, mongodb_default_socket_timeout))) {

610 - backend_set_mongodb_variables(&default_port, &backend_response_checker, &backend_request_formatter);

611 - do_mongodb = 1;

612 - }

613 - else {

614 - error("BACKEND: cannot initialize MongoDB backend");

615 - goto cleanup;

616 - }

617 -#else

618 - error("BACKEND: MongoDB support isn't compiled");

619 -#endif // HAVE_MONGOC

620 - break;

621 - }

622 - case BACKEND_TYPE_GRAPHITE: {

623 - backend_set_graphite_variables(&default_port,&backend_response_checker,&backend_request_formatter);

624 - break;

625 - }

626 - case BACKEND_TYPE_OPENTSDB_USING_TELNET: {

627 - backend_set_opentsdb_telnet_variables(&default_port,&backend_response_checker,&backend_request_formatter);

628 - break;

629 - }

630 - case BACKEND_TYPE_JSON: {

631 - backend_set_json_variables(&default_port,&backend_response_checker,&backend_request_formatter);

632 - break;

633 - }

634 - case BACKEND_TYPE_UNKNOWN: {

635 - break;

636 - }

637 - default: {

638 - break;

639 - }

640 - }

641 -

642 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

643 - if((backend_request_formatter == NULL && !do_prometheus_remote_write) || backend_response_checker == NULL) {

644 -#else

645 - if(backend_request_formatter == NULL || backend_response_checker == NULL) {

646 -#endif

647 - error("BACKEND: backend is misconfigured - disabling it.");

648 - goto cleanup;

649 - }

650 -

651 -

652 -// ------------------------------------------------------------------------

653 -// prepare the charts for monitoring the backend operation

654 -

655 - struct rusage thread;

656 -

657 - collected_number

658 - chart_buffered_metrics = 0,

659 - chart_lost_metrics = 0,

660 - chart_sent_metrics = 0,

661 - chart_buffered_bytes = 0,

662 - chart_received_bytes = 0,

663 - chart_sent_bytes = 0,

664 - chart_receptions = 0,

665 - chart_transmission_successes = 0,

666 - chart_transmission_failures = 0,

667 - chart_data_lost_events = 0,

668 - chart_lost_bytes = 0,

669 - chart_backend_reconnects = 0;

670 - // chart_backend_latency = 0;

671 -

672 - RRDSET *chart_metrics = rrdset_create_localhost("netdata", "backend_metrics", NULL, "backend", NULL, "Netdata Buffered Metrics", "metrics", "backends", NULL, 130600, global_backend_update_every, RRDSET_TYPE_LINE);

673 - rrddim_add(chart_metrics, "buffered", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

674 - rrddim_add(chart_metrics, "lost", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

675 - rrddim_add(chart_metrics, "sent", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

676 -

677 - RRDSET *chart_bytes = rrdset_create_localhost("netdata", "backend_bytes", NULL, "backend", NULL, "Netdata Backend Data Size", "KiB", "backends", NULL, 130610, global_backend_update_every, RRDSET_TYPE_AREA);

678 - rrddim_add(chart_bytes, "buffered", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

679 - rrddim_add(chart_bytes, "lost", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

680 - rrddim_add(chart_bytes, "sent", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

681 - rrddim_add(chart_bytes, "received", NULL, 1, 1024, RRD_ALGORITHM_ABSOLUTE);

682 -

683 - RRDSET *chart_ops = rrdset_create_localhost("netdata", "backend_ops", NULL, "backend", NULL, "Netdata Backend Operations", "operations", "backends", NULL, 130630, global_backend_update_every, RRDSET_TYPE_LINE);

684 - rrddim_add(chart_ops, "write", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

685 - rrddim_add(chart_ops, "discard", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

686 - rrddim_add(chart_ops, "reconnect", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

687 - rrddim_add(chart_ops, "failure", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

688 - rrddim_add(chart_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

689 -

690 - /*

691 - * this is misleading - we can only measure the time we need to send data

692 - * this time is not related to the time required for the data to travel to

693 - * the backend database and the time that server needed to process them

694 - *

695 - * issue #1432 and https://www.softlab.ntua.gr/facilities/documentation/unix/unix-socket-faq/unix-socket-faq-2.html

696 - *

697 - RRDSET *chart_latency = rrdset_create_localhost("netdata", "backend_latency", NULL, "backend", NULL, "Netdata Backend Latency", "ms", "backends", NULL, 130620, global_backend_update_every, RRDSET_TYPE_AREA);

698 - rrddim_add(chart_latency, "latency", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);

699 - */

700 -

701 - RRDSET *chart_rusage = rrdset_create_localhost("netdata", "backend_thread_cpu", NULL, "backend", NULL, "Netdata Backend Thread CPU usage", "milliseconds/s", "backends", NULL, 130630, global_backend_update_every, RRDSET_TYPE_STACKED);

702 - rrddim_add(chart_rusage, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);

703 - rrddim_add(chart_rusage, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);

704 -

705 -

706 - // ------------------------------------------------------------------------

707 - // prepare the backend main loop

708 -

709 - info("BACKEND: configured ('%s' on '%s' sending '%s' data, every %d seconds, as host '%s', with prefix '%s')", type, destination, source, global_backend_update_every, hostname, global_backend_prefix);

710 - send_statistics("BACKEND_START", "OK", type);

711 -

712 - usec_t step_ut = global_backend_update_every * USEC_PER_SEC;

713 - time_t after = now_realtime_sec();

714 - int failures = 0;

715 - heartbeat_t hb;

716 - heartbeat_init(&hb);

717 -

718 - while(!netdata_exit) {

719 -

720 - // ------------------------------------------------------------------------

721 - // Wait for the next iteration point.

722 -

723 - heartbeat_next(&hb, step_ut);

724 - time_t before = now_realtime_sec();

725 - debug(D_BACKEND, "BACKEND: preparing buffer for timeframe %lu to %lu", (unsigned long)after, (unsigned long)before);

726 -

727 - // ------------------------------------------------------------------------

728 - // add to the buffer the data we need to send to the backend

729 -

730 - netdata_thread_disable_cancelability();

731 -

732 - size_t count_hosts = 0;

733 - size_t count_charts_total = 0;

734 - size_t count_dims_total = 0;

735 -

736 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

737 - if(do_prometheus_remote_write)

738 - backends_clear_write_request();

739 -#endif

740 - rrd_rdlock();

741 - RRDHOST *host;

742 - rrdhost_foreach_read(host) {

743 - if(unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_BACKEND_SEND|RRDHOST_FLAG_BACKEND_DONT_SEND))) {

744 - char *name = (host == localhost)?"localhost":host->hostname;

745 - if (!hosts_pattern || simple_pattern_matches(hosts_pattern, name)) {

746 - rrdhost_flag_set(host, RRDHOST_FLAG_BACKEND_SEND);

747 - info("enabled backend for host '%s'", name);

748 - }

749 - else {

750 - rrdhost_flag_set(host, RRDHOST_FLAG_BACKEND_DONT_SEND);

751 - info("disabled backend for host '%s'", name);

752 - }

753 - }

754 -

755 - if(unlikely(!rrdhost_flag_check(host, RRDHOST_FLAG_BACKEND_SEND)))

756 - continue;

757 -

758 - rrdhost_rdlock(host);

759 -

760 - count_hosts++;

761 - size_t count_charts = 0;

762 - size_t count_dims = 0;

763 - size_t count_dims_skipped = 0;

764 -

765 - const char *__hostname = (host == localhost)?hostname:host->hostname;

766 -

767 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

768 - if(do_prometheus_remote_write) {

769 - backends_rrd_stats_remote_write_allmetrics_prometheus(

770 - host

771 - , __hostname

772 - , global_backend_prefix

773 - , global_backend_options

774 - , after

775 - , before

776 - , &count_charts

777 - , &count_dims

778 - , &count_dims_skipped

779 - );

780 - chart_buffered_metrics += count_dims;

781 - }

782 - else

783 -#endif

784 - {

785 - RRDSET *st;

786 - rrdset_foreach_read(st, host) {

787 - if(likely(backends_can_send_rrdset(global_backend_options, st))) {

788 - rrdset_rdlock(st);

789 -

790 - count_charts++;

791 -

792 - RRDDIM *rd;

793 - rrddim_foreach_read(rd, st) {

794 - if (likely(rd->last_collected_time.tv_sec >= after)) {

795 - chart_buffered_metrics += backend_request_formatter(b, global_backend_prefix, host, __hostname, st, rd, after, before, global_backend_options);

796 - count_dims++;

797 - }

798 - else {

799 - debug(D_BACKEND, "BACKEND: not sending dimension '%s' of chart '%s' from host '%s', its last data collection (%lu) is not within our timeframe (%lu to %lu)", rd->id, st->id, __hostname, (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)after, (unsigned long)before);

800 - count_dims_skipped++;

801 - }

802 - }

803 -

804 - rrdset_unlock(st);

805 - }

806 - }

807 - }

808 -

809 - debug(D_BACKEND, "BACKEND: sending host '%s', metrics of %zu dimensions, of %zu charts. Skipped %zu dimensions.", __hostname, count_dims, count_charts, count_dims_skipped);

810 - count_charts_total += count_charts;

811 - count_dims_total += count_dims;

812 -

813 - rrdhost_unlock(host);

814 - }

815 - rrd_unlock();

816 -

817 - netdata_thread_enable_cancelability();

818 -

819 - debug(D_BACKEND, "BACKEND: buffer has %zu bytes, added metrics for %zu dimensions, of %zu charts, from %zu hosts", buffer_strlen(b), count_dims_total, count_charts_total, count_hosts);

820 -

821 - // ------------------------------------------------------------------------

822 -

823 - chart_buffered_bytes = (collected_number)buffer_strlen(b);

824 -

825 - // reset the monitoring chart counters

826 - chart_received_bytes =

827 - chart_sent_bytes =

828 - chart_sent_metrics =

829 - chart_lost_metrics =

830 - chart_receptions =

831 - chart_transmission_successes =

832 - chart_transmission_failures =

833 - chart_data_lost_events =

834 - chart_lost_bytes =

835 - chart_backend_reconnects = 0;

836 - // chart_backend_latency = 0;

837 -

838 - if(unlikely(netdata_exit)) break;

839 -

840 - //fprintf(stderr, "\nBACKEND BEGIN:\n%s\nBACKEND END\n", buffer_tostring(b));

841 - //fprintf(stderr, "after = %lu, before = %lu\n", after, before);

842 -

843 - // prepare for the next iteration

844 - // to add incrementally data to buffer

845 - after = before;

846 -

847 -#if HAVE_KINESIS

848 - if(do_kinesis) {

849 - unsigned long long partition_key_seq = 0;

850 -

851 - size_t buffer_len = buffer_strlen(b);

852 - size_t sent = 0;

853 -

854 - while(sent < buffer_len) {

855 - char partition_key[KINESIS_PARTITION_KEY_MAX + 1];

856 - snprintf(partition_key, KINESIS_PARTITION_KEY_MAX, "netdata_%llu", partition_key_seq++);

857 - size_t partition_key_len = strnlen(partition_key, KINESIS_PARTITION_KEY_MAX);

858 -

859 - const char *first_char = buffer_tostring(b) + sent;

860 -

861 - size_t record_len = 0;

862 -

863 - // split buffer into chunks of maximum allowed size

864 - if(buffer_len - sent < KINESIS_RECORD_MAX - partition_key_len) {

865 - record_len = buffer_len - sent;

866 - }

867 - else {

868 - record_len = KINESIS_RECORD_MAX - partition_key_len;

869 - while(*(first_char + record_len) != '\n' && record_len) record_len--;

870 - }

871 -

872 - char error_message[ERROR_LINE_MAX + 1] = "";

873 -

874 - debug(D_BACKEND, "BACKEND: backends_kinesis_put_record(): dest = %s, id = %s, key = %s, stream = %s, partition_key = %s, \

875 - buffer = %zu, record = %zu", destination, kinesis_auth_key_id, kinesis_secure_key, kinesis_stream_name,

876 - partition_key, buffer_len, record_len);

877 -

878 - backends_kinesis_put_record(kinesis_stream_name, partition_key, first_char, record_len);

879 -

880 - sent += record_len;

881 - chart_transmission_successes++;

882 -

883 - size_t sent_bytes = 0, lost_bytes = 0;

884 -

885 - if(unlikely(backends_kinesis_get_result(error_message, &sent_bytes, &lost_bytes))) {

886 - // oops! we couldn't send (all or some of the) data

887 - error("BACKEND: %s", error_message);

888 - error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zu bytes.",

889 - destination, sent_bytes, sent_bytes - lost_bytes);

890 -

891 - chart_transmission_failures++;

892 - chart_data_lost_events++;

893 - chart_lost_bytes += lost_bytes;

894 -

895 - // estimate the number of lost metrics

896 - chart_lost_metrics += (collected_number)(chart_buffered_metrics

897 - * (buffer_len && (lost_bytes > buffer_len) ? (double)lost_bytes / buffer_len : 1));

898 -

899 - break;

900 - }

901 - else {

902 - chart_receptions++;

903 - }

904 -

905 - if(unlikely(netdata_exit)) break;

906 - }

907 -

908 - chart_sent_bytes += sent;

909 - if(likely(sent == buffer_len))

910 - chart_sent_metrics = chart_buffered_metrics;

911 -

912 - buffer_flush(b);

913 - } else

914 -#endif /* HAVE_KINESIS */

915 -

916 -#if HAVE_MONGOC

917 - if(do_mongodb) {

918 - size_t buffer_len = buffer_strlen(b);

919 - size_t sent = 0;

920 -

921 - while(sent < buffer_len) {

922 - const char *first_char = buffer_tostring(b);

923 -

924 - debug(D_BACKEND, "BACKEND: backends_mongodb_insert(): uri = %s, database = %s, collection = %s, \

925 - buffer = %zu", mongodb_uri, mongodb_database, mongodb_collection, buffer_len);

926 -

927 - if(likely(!backends_mongodb_insert((char *)first_char, (size_t)chart_buffered_metrics))) {

928 - sent += buffer_len;

929 - chart_transmission_successes++;

930 - chart_receptions++;

931 - }

932 - else {

933 - // oops! we couldn't send (all or some of the) data

934 - error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zu bytes.",

935 - mongodb_uri, buffer_len, 0UL);

936 -

937 - chart_transmission_failures++;

938 - chart_data_lost_events++;

939 - chart_lost_bytes += buffer_len;

940 -

941 - // estimate the number of lost metrics

942 - chart_lost_metrics += (collected_number)chart_buffered_metrics;

943 -

944 - break;

945 - }

946 -

947 - if(unlikely(netdata_exit)) break;

948 - }

949 -

950 - chart_sent_bytes += sent;

951 - if(likely(sent == buffer_len))

952 - chart_sent_metrics = chart_buffered_metrics;

953 -

954 - buffer_flush(b);

955 - } else

956 -#endif /* HAVE_MONGOC */

957 -

958 - {

959 -

960 - // ------------------------------------------------------------------------

961 - // if we are connected, receive a response, without blocking

962 -

963 - if(likely(sock != -1)) {

964 - errno = 0;

965 -

966 - // loop through to collect all data

967 - while(sock != -1 && errno != EWOULDBLOCK) {

968 - buffer_need_bytes(response, 4096);

969 -

970 - ssize_t r;

971 -#ifdef ENABLE_HTTPS

972 - if(opentsdb_ssl.conn && !opentsdb_ssl.flags) {

973 - r = SSL_read(opentsdb_ssl.conn, &response->buffer[response->len], response->size - response->len);

974 - } else {

975 - r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);

976 - }

977 -#else

978 - r = recv(sock, &response->buffer[response->len], response->size - response->len, MSG_DONTWAIT);

979 -#endif

980 - if(likely(r > 0)) {

981 - // we received some data

982 - response->len += r;

983 - chart_received_bytes += r;

984 - chart_receptions++;

985 - }

986 - else if(r == 0) {

987 - error("BACKEND: '%s' closed the socket", destination);

988 - close(sock);

989 - sock = -1;

990 - }

991 - else {

992 - // failed to receive data

993 - if(errno != EAGAIN && errno != EWOULDBLOCK) {

994 - error("BACKEND: cannot receive data from backend '%s'.", destination);

995 - }

996 - }

997 - }

998 -

999 - // if we received data, process them

1000 - if(buffer_strlen(response))

1001 - backend_response_checker(response);

1002 - }

1003 -

1004 - // ------------------------------------------------------------------------

1005 - // if we are not connected, connect to a backend server

1006 -

1007 - if(unlikely(sock == -1)) {

1008 - // usec_t start_ut = now_monotonic_usec();

1009 - size_t reconnects = 0;

1010 -

1011 - sock = connect_to_one_of(destination, default_port, &timeout, &reconnects, NULL, 0);

1012 -#ifdef ENABLE_HTTPS

1013 - if(sock != -1) {

1014 - if(netdata_exporting_ctx) {

1015 - if(!opentsdb_ssl.conn) {

1016 - opentsdb_ssl.conn = SSL_new(netdata_exporting_ctx);

1017 - if(!opentsdb_ssl.conn) {

1018 - error("Failed to allocate SSL structure %d.", sock);

1019 - opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;

1020 - }

1021 - } else {

1022 - SSL_clear(opentsdb_ssl.conn);

1023 - }

1024 - }

1025 -

1026 - if(opentsdb_ssl.conn) {

1027 - if(SSL_set_fd(opentsdb_ssl.conn, sock) != 1) {

1028 - error("Failed to set the socket to the SSL on socket fd %d.", host->rrdpush_sender_socket);

1029 - opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;

1030 - } else {

1031 - opentsdb_ssl.flags = NETDATA_SSL_HANDSHAKE_COMPLETE;

1032 - SSL_set_connect_state(opentsdb_ssl.conn);

1033 - int err = SSL_connect(opentsdb_ssl.conn);

1034 - if (err != 1) {

1035 - err = SSL_get_error(opentsdb_ssl.conn, err);

1036 - error("SSL cannot connect with the server: %s ", ERR_error_string((long)SSL_get_error(opentsdb_ssl.conn, err), NULL));

1037 - opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;

1038 - } //TODO: check certificate here

1039 - }

1040 - }

1041 - }

1042 -#endif

1043 - chart_backend_reconnects += reconnects;

1044 - // chart_backend_latency += now_monotonic_usec() - start_ut;

1045 - }

1046 -

1047 - if(unlikely(netdata_exit)) break;

1048 -

1049 - // ------------------------------------------------------------------------

1050 - // if we are connected, send our buffer to the backend server

1051 -

1052 - if(likely(sock != -1)) {

1053 - size_t len = buffer_strlen(b);

1054 - // usec_t start_ut = now_monotonic_usec();

1055 - int flags = 0;

1056 - #ifdef MSG_NOSIGNAL

1057 - flags += MSG_NOSIGNAL;

1058 - #endif

1059 -

1060 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

1061 - if(do_prometheus_remote_write) {

1062 - size_t data_size = backends_get_write_request_size();

1063 -

1064 - if(unlikely(!data_size)) {

1065 - error("BACKEND: write request size is out of range");

1066 - continue;

1067 - }

1068 -

1069 - buffer_flush(b);

1070 - buffer_need_bytes(b, data_size);

1071 - if(unlikely(backends_pack_write_request(b->buffer, &data_size))) {

1072 - error("BACKEND: cannot pack write request");

1073 - continue;

1074 - }

1075 - b->len = data_size;

1076 - chart_buffered_bytes = (collected_number)buffer_strlen(b);

1077 -

1078 - buffer_flush(http_request_header);

1079 - buffer_sprintf(http_request_header,

1080 - "POST %s HTTP/1.1\r\n"

1081 - "Host: %s\r\n"

1082 - "Accept: */*\r\n"

1083 - "Content-Length: %zu\r\n"

1084 - "Content-Type: application/x-www-form-urlencoded\r\n\r\n",

1085 - remote_write_path,

1086 - destination,

1087 - data_size

1088 - );

1089 -

1090 - len = buffer_strlen(http_request_header);

1091 - send(sock, buffer_tostring(http_request_header), len, flags);

1092 -

1093 - len = data_size;

1094 - }

1095 -#endif

1096 -

1097 - ssize_t written;

1098 -#ifdef ENABLE_HTTPS

1099 - if(opentsdb_ssl.conn && !opentsdb_ssl.flags) {

1100 - written = SSL_write(opentsdb_ssl.conn, buffer_tostring(b), len);

1101 - } else {

1102 - written = send(sock, buffer_tostring(b), len, flags);

1103 - }

1104 -#else

1105 - written = send(sock, buffer_tostring(b), len, flags);

1106 -#endif

1107 -

1108 - // chart_backend_latency += now_monotonic_usec() - start_ut;

1109 - if(written != -1 && (size_t)written == len) {

1110 - // we sent the data successfully

1111 - chart_transmission_successes++;

1112 - chart_sent_bytes += written;

1113 - chart_sent_metrics = chart_buffered_metrics;

1114 -

1115 - // reset the failures count

1116 - failures = 0;

1117 -

1118 - // empty the buffer

1119 - buffer_flush(b);

1120 - }

1121 - else {

1122 - // oops! we couldn't send (all or some of the) data

1123 - error("BACKEND: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.", destination, len, written);

1124 - chart_transmission_failures++;

1125 -

1126 - if(written != -1)

1127 - chart_sent_bytes += written;

1128 -

1129 - // increment the counter we check for data loss

1130 - failures++;

1131 -

1132 - // close the socket - we will re-open it next time

1133 - close(sock);

1134 - sock = -1;

1135 - }

1136 - }

1137 - else {

1138 - error("BACKEND: failed to update database backend '%s'", destination);

1139 - chart_transmission_failures++;

1140 -

1141 - // increment the counter we check for data loss

1142 - failures++;

1143 - }

1144 - }

1145 -

1146 -

1147 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

1148 - if(do_prometheus_remote_write && failures) {

1149 - (void) buffer_on_failures;

1150 - failures = 0;

1151 - chart_lost_bytes = chart_buffered_bytes = backends_get_write_request_size(); // estimated write request size

1152 - chart_data_lost_events++;

1153 - chart_lost_metrics = chart_buffered_metrics;

1154 - } else

1155 -#endif

1156 - if(failures > buffer_on_failures) {

1157 - // too bad! we are going to lose data

1158 - chart_lost_bytes += buffer_strlen(b);

1159 - error("BACKEND: reached %d backend failures. Flushing buffers to protect this host - this results in data loss on back-end server '%s'", failures, destination);

1160 - buffer_flush(b);

1161 - failures = 0;

1162 - chart_data_lost_events++;

1163 - chart_lost_metrics = chart_buffered_metrics;

1164 - }

1165 -

1166 - if(unlikely(netdata_exit)) break;

1167 -

1168 - // ------------------------------------------------------------------------

1169 - // update the monitoring charts

1170 -

1171 - if(likely(chart_ops->counter_done)) rrdset_next(chart_ops);

1172 - rrddim_set(chart_ops, "read", chart_receptions);

1173 - rrddim_set(chart_ops, "write", chart_transmission_successes);

1174 - rrddim_set(chart_ops, "discard", chart_data_lost_events);

1175 - rrddim_set(chart_ops, "failure", chart_transmission_failures);

1176 - rrddim_set(chart_ops, "reconnect", chart_backend_reconnects);

1177 - rrdset_done(chart_ops);

1178 -

1179 - if(likely(chart_metrics->counter_done)) rrdset_next(chart_metrics);

1180 - rrddim_set(chart_metrics, "buffered", chart_buffered_metrics);

1181 - rrddim_set(chart_metrics, "lost", chart_lost_metrics);

1182 - rrddim_set(chart_metrics, "sent", chart_sent_metrics);

1183 - rrdset_done(chart_metrics);

1184 -

1185 - if(likely(chart_bytes->counter_done)) rrdset_next(chart_bytes);

1186 - rrddim_set(chart_bytes, "buffered", chart_buffered_bytes);

1187 - rrddim_set(chart_bytes, "lost", chart_lost_bytes);

1188 - rrddim_set(chart_bytes, "sent", chart_sent_bytes);

1189 - rrddim_set(chart_bytes, "received", chart_received_bytes);

1190 - rrdset_done(chart_bytes);

1191 -

1192 - /*

1193 - if(likely(chart_latency->counter_done)) rrdset_next(chart_latency);

1194 - rrddim_set(chart_latency, "latency", chart_backend_latency);

1195 - rrdset_done(chart_latency);

1196 - */

1197 -

1198 - getrusage(RUSAGE_THREAD, &thread);

1199 - if(likely(chart_rusage->counter_done)) rrdset_next(chart_rusage);

1200 - rrddim_set(chart_rusage, "user", thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);

1201 - rrddim_set(chart_rusage, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);

1202 - rrdset_done(chart_rusage);

1203 -

1204 - if(likely(buffer_strlen(b) == 0))

1205 - chart_buffered_metrics = 0;

1206 -

1207 - if(unlikely(netdata_exit)) break;

1208 - }

1209 -

1210 -cleanup:

1211 -#if HAVE_KINESIS

1212 - if(do_kinesis) {

1213 - backends_kinesis_shutdown();

1214 - freez(kinesis_auth_key_id);

1215 - freez(kinesis_secure_key);

1216 - freez(kinesis_stream_name);

1217 - }

1218 -#endif

1219 -

1220 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

1221 - buffer_free(http_request_header);

1222 - if(do_prometheus_remote_write)

1223 - backends_protocol_buffers_shutdown();

1224 -#endif

1225 -

1226 -#if HAVE_MONGOC

1227 - if(do_mongodb) {

1228 - backends_mongodb_cleanup();

1229 - freez(mongodb_uri);

1230 - freez(mongodb_database);

1231 - freez(mongodb_collection);

1232 - }

1233 -#endif

1234 -

1235 - if(sock != -1)

1236 - close(sock);

1237 -

1238 - buffer_free(b);

1239 - buffer_free(response);

1240 -

1241 -#ifdef ENABLE_HTTPS

1242 - if(netdata_exporting_ctx) {

1243 - if(opentsdb_ssl.conn) {

1244 - SSL_free(opentsdb_ssl.conn);

1245 - }

1246 - }

1247 -#endif

1248 -

1249 - netdata_thread_cleanup_pop(1);

1250 - return NULL;

1251 -}

backends/backends.h deleted

-98

@@ -1,98 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_BACKENDS_H

4 -#define NETDATA_BACKENDS_H 1

5 -

6 -#include "daemon/common.h"

7 -

8 -typedef enum backend_options {

9 - BACKEND_OPTION_NONE = 0,

10 -

11 - BACKEND_SOURCE_DATA_AS_COLLECTED = (1 << 0),

12 - BACKEND_SOURCE_DATA_AVERAGE = (1 << 1),

13 - BACKEND_SOURCE_DATA_SUM = (1 << 2),

14 -

15 - BACKEND_OPTION_SEND_NAMES = (1 << 16)

16 -} BACKEND_OPTIONS;

17 -

18 -typedef enum backend_types {

19 - BACKEND_TYPE_UNKNOWN, // Invalid type

20 - BACKEND_TYPE_GRAPHITE, // Send plain text to Graphite

21 - BACKEND_TYPE_OPENTSDB_USING_TELNET, // Send data to OpenTSDB using telnet API

22 - BACKEND_TYPE_OPENTSDB_USING_HTTP, // Send data to OpenTSDB using HTTP API

23 - BACKEND_TYPE_JSON, // Stores the data using JSON.

24 - BACKEND_TYPE_PROMETHEUS_REMOTE_WRITE, // The user selected to use Prometheus backend

25 - BACKEND_TYPE_KINESIS, // Send message to AWS Kinesis

26 - BACKEND_TYPE_MONGODB, // Send data to MongoDB collection

27 - BACKEND_TYPE_NUM // Number of backend types

28 -} BACKEND_TYPE;

29 -

30 -typedef int (**backend_response_checker_t)(BUFFER *);

31 -typedef int (**backend_request_formatter_t)(BUFFER *, const char *, RRDHOST *, const char *, RRDSET *, RRDDIM *, time_t, time_t, BACKEND_OPTIONS);

32 -

33 -#define BACKEND_OPTIONS_SOURCE_BITS (BACKEND_SOURCE_DATA_AS_COLLECTED|BACKEND_SOURCE_DATA_AVERAGE|BACKEND_SOURCE_DATA_SUM)

34 -#define BACKEND_OPTIONS_DATA_SOURCE(backend_options) (backend_options & BACKEND_OPTIONS_SOURCE_BITS)

35 -

36 -extern int global_backend_update_every;

37 -extern BACKEND_OPTIONS global_backend_options;

38 -extern const char *global_backend_source;

39 -extern const char *global_backend_prefix;

40 -extern const char *global_backend_send_charts_matching;

41 -

42 -extern void *backends_main(void *ptr);

43 -BACKEND_TYPE backend_select_type(const char *type);

44 -

45 -extern BACKEND_OPTIONS backend_parse_data_source(const char *source, BACKEND_OPTIONS backend_options);

46 -

47 -#ifdef BACKENDS_INTERNALS

48 -

49 -extern int backends_can_send_rrdset(BACKEND_OPTIONS backend_options, RRDSET *st);

50 -extern calculated_number backend_calculate_value_from_stored_data(

51 - RRDSET *st // the chart

52 - , RRDDIM *rd // the dimension

53 - , time_t after // the start timestamp

54 - , time_t before // the end timestamp

55 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

56 - , time_t *first_timestamp // the timestamp of the first point used in this response

57 - , time_t *last_timestamp // the timestamp that should be reported to backend

58 -);

59 -

60 -extern size_t backend_name_copy(char *d, const char *s, size_t usable);

61 -extern int discard_response(BUFFER *b, const char *backend);

62 -

63 -static inline char *strip_quotes(char *str) {

64 - if(*str == '"' || *str == '\'') {

65 - char *s;

66 -

67 - str++;

68 -

69 - s = str;

70 - while(*s) s++;

71 - if(s != str) s--;

72 -

73 - if(*s == '"' || *s == '\'') *s = '\0';

74 - }

75 -

76 - return str;

77 -}

78 -

79 -#endif // BACKENDS_INTERNALS

80 -

81 -#include "backends/prometheus/backend_prometheus.h"

82 -#include "backends/graphite/graphite.h"

83 -#include "backends/json/json.h"

84 -#include "backends/opentsdb/opentsdb.h"

85 -

86 -#if HAVE_KINESIS

87 -#include "backends/aws_kinesis/aws_kinesis.h"

88 -#endif

89 -

90 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

91 -#include "backends/prometheus/remote_write/remote_write.h"

92 -#endif

93 -

94 -#if HAVE_MONGOC

95 -#include "backends/mongodb/mongodb.h"

96 -#endif

97 -

98 -#endif /* NETDATA_BACKENDS_H */

backends/graphite/Makefile.am deleted

-4

@@ -1,4 +0,0 @@

1 -# SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -AUTOMAKE_OPTIONS = subdir-objects

4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in

backends/graphite/graphite.c deleted

-90

@@ -1,90 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#define BACKENDS_INTERNALS

4 -#include "graphite.h"

5 -

6 -// ----------------------------------------------------------------------------

7 -// graphite backend

8 -

9 -int backends_format_dimension_collected_graphite_plaintext(

10 - BUFFER *b // the buffer to write data to

11 - , const char *prefix // the prefix to use

12 - , RRDHOST *host // the host this chart comes from

13 - , const char *hostname // the hostname (to override host->hostname)

14 - , RRDSET *st // the chart

15 - , RRDDIM *rd // the dimension

16 - , time_t after // the start timestamp

17 - , time_t before // the end timestamp

18 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

19 -) {

20 - (void)host;

21 - (void)after;

22 - (void)before;

23 -

24 - char chart_name[RRD_ID_LENGTH_MAX + 1];

25 - char dimension_name[RRD_ID_LENGTH_MAX + 1];

26 - backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

27 - backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

28 -

29 - buffer_sprintf(

30 - b

31 - , "%s.%s.%s.%s%s%s " COLLECTED_NUMBER_FORMAT " %llu\n"

32 - , prefix

33 - , hostname

34 - , chart_name

35 - , dimension_name

36 - , (host->tags)?";":""

37 - , (host->tags)?host->tags:""

38 - , rd->last_collected_value

39 - , (unsigned long long)rd->last_collected_time.tv_sec

40 - );

41 -

42 - return 1;

43 -}

44 -

45 -int backends_format_dimension_stored_graphite_plaintext(

46 - BUFFER *b // the buffer to write data to

47 - , const char *prefix // the prefix to use

48 - , RRDHOST *host // the host this chart comes from

49 - , const char *hostname // the hostname (to override host->hostname)

50 - , RRDSET *st // the chart

51 - , RRDDIM *rd // the dimension

52 - , time_t after // the start timestamp

53 - , time_t before // the end timestamp

54 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

55 -) {

56 - (void)host;

57 -

58 - char chart_name[RRD_ID_LENGTH_MAX + 1];

59 - char dimension_name[RRD_ID_LENGTH_MAX + 1];

60 - backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

61 - backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

62 -

63 - time_t first_t = after, last_t = before;

64 - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);

65 -

66 - if(!isnan(value)) {

67 -

68 - buffer_sprintf(

69 - b

70 - , "%s.%s.%s.%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n"

71 - , prefix

72 - , hostname

73 - , chart_name

74 - , dimension_name

75 - , (host->tags)?";":""

76 - , (host->tags)?host->tags:""

77 - , value

78 - , (unsigned long long) last_t

79 - );

80 -

81 - return 1;

82 - }

83 - return 0;

84 -}

85 -

86 -int process_graphite_response(BUFFER *b) {

87 - return discard_response(b, "graphite");

88 -}

89 -

90 -

backends/graphite/graphite.h deleted

-35

@@ -1,35 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -

4 -#ifndef NETDATA_BACKEND_GRAPHITE_H

5 -#define NETDATA_BACKEND_GRAPHITE_H

6 -

7 -#include "backends/backends.h"

8 -

9 -extern int backends_format_dimension_collected_graphite_plaintext(

10 - BUFFER *b // the buffer to write data to

11 - , const char *prefix // the prefix to use

12 - , RRDHOST *host // the host this chart comes from

13 - , const char *hostname // the hostname (to override host->hostname)

14 - , RRDSET *st // the chart

15 - , RRDDIM *rd // the dimension

16 - , time_t after // the start timestamp

17 - , time_t before // the end timestamp

18 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

19 -);

20 -

21 -extern int backends_format_dimension_stored_graphite_plaintext(

22 - BUFFER *b // the buffer to write data to

23 - , const char *prefix // the prefix to use

24 - , RRDHOST *host // the host this chart comes from

25 - , const char *hostname // the hostname (to override host->hostname)

26 - , RRDSET *st // the chart

27 - , RRDDIM *rd // the dimension

28 - , time_t after // the start timestamp

29 - , time_t before // the end timestamp

30 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

31 -);

32 -

33 -extern int process_graphite_response(BUFFER *b);

34 -

35 -#endif //NETDATA_BACKEND_GRAPHITE_H

backends/json/Makefile.am deleted

-4

@@ -1,4 +0,0 @@

1 -# SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -AUTOMAKE_OPTIONS = subdir-objects

4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in

backends/json/json.c deleted

-152

@@ -1,152 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#define BACKENDS_INTERNALS

4 -#include "json.h"

5 -

6 -// ----------------------------------------------------------------------------

7 -// json backend

8 -

9 -int backends_format_dimension_collected_json_plaintext(

10 - BUFFER *b // the buffer to write data to

11 - , const char *prefix // the prefix to use

12 - , RRDHOST *host // the host this chart comes from

13 - , const char *hostname // the hostname (to override host->hostname)

14 - , RRDSET *st // the chart

15 - , RRDDIM *rd // the dimension

16 - , time_t after // the start timestamp

17 - , time_t before // the end timestamp

18 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

19 -) {

20 - (void)host;

21 - (void)after;

22 - (void)before;

23 - (void)backend_options;

24 -

25 - const char *tags_pre = "", *tags_post = "", *tags = host->tags;

26 - if(!tags) tags = "";

27 -

28 - if(*tags) {

29 - if(*tags == '{' || *tags == '[' || *tags == '"') {

30 - tags_pre = "\"host_tags\":";

31 - tags_post = ",";

32 - }

33 - else {

34 - tags_pre = "\"host_tags\":\"";

35 - tags_post = "\",";

36 - }

37 - }

38 -

39 - buffer_sprintf(b, "{"

40 - "\"prefix\":\"%s\","

41 - "\"hostname\":\"%s\","

42 - "%s%s%s"

43 -

44 - "\"chart_id\":\"%s\","

45 - "\"chart_name\":\"%s\","

46 - "\"chart_family\":\"%s\","

47 - "\"chart_context\": \"%s\","

48 - "\"chart_type\":\"%s\","

49 - "\"units\": \"%s\","

50 -

51 - "\"id\":\"%s\","

52 - "\"name\":\"%s\","

53 - "\"value\":" COLLECTED_NUMBER_FORMAT ","

54 -

55 - "\"timestamp\": %llu}\n",

56 - prefix,

57 - hostname,

58 - tags_pre, tags, tags_post,

59 -

60 - st->id,

61 - st->name,

62 - st->family,

63 - st->context,

64 - st->type,

65 - st->units,

66 -

67 - rd->id,

68 - rd->name,

69 - rd->last_collected_value,

70 -

71 - (unsigned long long) rd->last_collected_time.tv_sec

72 - );

73 -

74 - return 1;

75 -}

76 -

77 -int backends_format_dimension_stored_json_plaintext(

78 - BUFFER *b // the buffer to write data to

79 - , const char *prefix // the prefix to use

80 - , RRDHOST *host // the host this chart comes from

81 - , const char *hostname // the hostname (to override host->hostname)

82 - , RRDSET *st // the chart

83 - , RRDDIM *rd // the dimension

84 - , time_t after // the start timestamp

85 - , time_t before // the end timestamp

86 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

87 -) {

88 - (void)host;

89 -

90 - time_t first_t = after, last_t = before;

91 - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);

92 -

93 - if(!isnan(value)) {

94 - const char *tags_pre = "", *tags_post = "", *tags = host->tags;

95 - if(!tags) tags = "";

96 -

97 - if(*tags) {

98 - if(*tags == '{' || *tags == '[' || *tags == '"') {

99 - tags_pre = "\"host_tags\":";

100 - tags_post = ",";

101 - }

102 - else {

103 - tags_pre = "\"host_tags\":\"";

104 - tags_post = "\",";

105 - }

106 - }

107 -

108 - buffer_sprintf(b, "{"

109 - "\"prefix\":\"%s\","

110 - "\"hostname\":\"%s\","

111 - "%s%s%s"

112 -

113 - "\"chart_id\":\"%s\","

114 - "\"chart_name\":\"%s\","

115 - "\"chart_family\":\"%s\","

116 - "\"chart_context\": \"%s\","

117 - "\"chart_type\":\"%s\","

118 - "\"units\": \"%s\","

119 -

120 - "\"id\":\"%s\","

121 - "\"name\":\"%s\","

122 - "\"value\":" CALCULATED_NUMBER_FORMAT ","

123 -

124 - "\"timestamp\": %llu}\n",

125 - prefix,

126 - hostname,

127 - tags_pre, tags, tags_post,

128 -

129 - st->id,

130 - st->name,

131 - st->family,

132 - st->context,

133 - st->type,

134 - st->units,

135 -

136 - rd->id,

137 - rd->name,

138 - value,

139 -

140 - (unsigned long long) last_t

141 - );

142 -

143 - return 1;

144 - }

145 - return 0;

146 -}

147 -

148 -int process_json_response(BUFFER *b) {

149 - return discard_response(b, "json");

150 -}

151 -

152 -

backends/json/json.h deleted

-34

@@ -1,34 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_BACKEND_JSON_H

4 -#define NETDATA_BACKEND_JSON_H

5 -

6 -#include "backends/backends.h"

7 -

8 -extern int backends_format_dimension_collected_json_plaintext(

9 - BUFFER *b // the buffer to write data to

10 - , const char *prefix // the prefix to use

11 - , RRDHOST *host // the host this chart comes from

12 - , const char *hostname // the hostname (to override host->hostname)

13 - , RRDSET *st // the chart

14 - , RRDDIM *rd // the dimension

15 - , time_t after // the start timestamp

16 - , time_t before // the end timestamp

17 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

18 -);

19 -

20 -extern int backends_format_dimension_stored_json_plaintext(

21 - BUFFER *b // the buffer to write data to

22 - , const char *prefix // the prefix to use

23 - , RRDHOST *host // the host this chart comes from

24 - , const char *hostname // the hostname (to override host->hostname)

25 - , RRDSET *st // the chart

26 - , RRDDIM *rd // the dimension

27 - , time_t after // the start timestamp

28 - , time_t before // the end timestamp

29 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

30 -);

31 -

32 -extern int process_json_response(BUFFER *b);

33 -

34 -#endif //NETDATA_BACKEND_JSON_H

backends/mongodb/Makefile.am deleted

-8

@@ -1,8 +0,0 @@

1 -# SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -AUTOMAKE_OPTIONS = subdir-objects

4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in

5 -

6 -dist_noinst_DATA = \

7 - README.md \

8 - $(NULL)

backends/mongodb/README.md deleted

-41

@@ -1,41 +0,0 @@

1 -<!--

2 -title: "MongoDB backend"

3 -custom_edit_url: https://github.com/netdata/netdata/edit/master/backends/mongodb/README.md

4 --->

5 -

6 -# MongoDB backend

7 -

8 -## Prerequisites

9 -

10 -To use MongoDB as a backend, `libmongoc` 1.7.0 or higher should be

11 -[installed](http://mongoc.org/libmongoc/current/installing.html) first. Next, Netdata should be re-installed from the

12 -source. The installer will detect that the required libraries are now available.

13 -

14 -## Configuration

15 -

16 -To enable data sending to the MongoDB backend set the following options in `netdata.conf`:

17 -

18 -```conf

19 -[backend]

20 - enabled = yes

21 - type = mongodb

22 -```

23 -

24 -In the Netdata configuration directory run `./edit-config mongodb.conf` and set [MongoDB

25 -URI](https://docs.mongodb.com/manual/reference/connection-string/), database name, and collection name:

26 -

27 -```yaml

28 -# URI

29 -uri = mongodb://<hostname>

30 -

31 -# database name

32 -database = your_database_name

33 -

34 -# collection name

35 -collection = your_collection_name

36 -```

37 -

38 -The default socket timeout depends on the backend update interval. The timeout is 500 ms shorter than the interval (but

39 -not less than 1000 ms). You can alter the timeout using the `sockettimeoutms` MongoDB URI option.

40 -

41 -

backends/mongodb/mongodb.c deleted

-189

@@ -1,189 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#define BACKENDS_INTERNALS

4 -#include "mongodb.h"

5 -#include <mongoc.h>

6 -

7 -#define CONFIG_FILE_LINE_MAX ((CONFIG_MAX_NAME + CONFIG_MAX_VALUE + 1024) * 2)

8 -

9 -static mongoc_client_t *mongodb_client;

10 -static mongoc_collection_t *mongodb_collection;

11 -

12 -int backends_mongodb_init(const char *uri_string,

13 - const char *database_string,

14 - const char *collection_string,

15 - int32_t default_socket_timeout) {

16 - mongoc_uri_t *uri;

17 - bson_error_t error;

18 -

19 - mongoc_init();

20 -

21 - uri = mongoc_uri_new_with_error(uri_string, &error);

22 - if(unlikely(!uri)) {

23 - error("BACKEND: failed to parse URI: %s. Error message: %s", uri_string, error.message);

24 - return 1;

25 - }

26 -

27 - int32_t socket_timeout = mongoc_uri_get_option_as_int32(uri, MONGOC_URI_SOCKETTIMEOUTMS, default_socket_timeout);

28 - if(!mongoc_uri_set_option_as_int32(uri, MONGOC_URI_SOCKETTIMEOUTMS, socket_timeout)) {

29 - error("BACKEND: failed to set %s to the value %d", MONGOC_URI_SOCKETTIMEOUTMS, socket_timeout);

30 - return 1;

31 - };

32 -

33 - mongodb_client = mongoc_client_new_from_uri(uri);

34 - if(unlikely(!mongodb_client)) {

35 - error("BACKEND: failed to create a new client");

36 - return 1;

37 - }

38 -

39 - if(!mongoc_client_set_appname(mongodb_client, "netdata")) {

40 - error("BACKEND: failed to set client appname");

41 - };

42 -

43 - mongodb_collection = mongoc_client_get_collection(mongodb_client, database_string, collection_string);

44 -

45 - mongoc_uri_destroy(uri);

46 -

47 - return 0;

48 -}

49 -

50 -void backends_free_bson(bson_t **insert, size_t n_documents) {

51 - size_t i;

52 -

53 - for(i = 0; i < n_documents; i++)

54 - bson_destroy(insert[i]);

55 -

56 - free(insert);

57 -}

58 -

59 -int backends_mongodb_insert(char *data, size_t n_metrics) {

60 - bson_t **insert = calloc(n_metrics, sizeof(bson_t *));

61 - bson_error_t error;

62 - char *start = data, *end = data;

63 - size_t n_documents = 0;

64 -

65 - while(*end && n_documents <= n_metrics) {

66 - while(*end && *end != '\n') end++;

67 -

68 - if(likely(*end)) {

69 - *end = '\0';

70 - end++;

71 - }

72 - else {

73 - break;

74 - }

75 -

76 - insert[n_documents] = bson_new_from_json((const uint8_t *)start, -1, &error);

77 -

78 - if(unlikely(!insert[n_documents])) {

79 - error("BACKEND: %s", error.message);

80 - backends_free_bson(insert, n_documents);

81 - return 1;

82 - }

83 -

84 - start = end;

85 -

86 - n_documents++;

87 - }

88 -

89 - if(unlikely(!mongoc_collection_insert_many(mongodb_collection, (const bson_t **)insert, n_documents, NULL, NULL, &error))) {

90 - error("BACKEND: %s", error.message);

91 - backends_free_bson(insert, n_documents);

92 - return 1;

93 - }

94 -

95 - backends_free_bson(insert, n_documents);

96 -

97 - return 0;

98 -}

99 -

100 -void backends_mongodb_cleanup() {

101 - mongoc_collection_destroy(mongodb_collection);

102 - mongoc_client_destroy(mongodb_client);

103 - mongoc_cleanup();

104 -

105 - return;

106 -}

107 -

108 -int read_mongodb_conf(const char *path, char **uri_p, char **database_p, char **collection_p) {

109 - char *uri = *uri_p;

110 - char *database = *database_p;

111 - char *collection = *collection_p;

112 -

113 - if(unlikely(uri)) freez(uri);

114 - if(unlikely(database)) freez(database);

115 - if(unlikely(collection)) freez(collection);

116 - uri = NULL;

117 - database = NULL;

118 - collection = NULL;

119 -

120 - int line = 0;

121 -

122 - char filename[FILENAME_MAX + 1];

123 - snprintfz(filename, FILENAME_MAX, "%s/mongodb.conf", path);

124 -

125 - char buffer[CONFIG_FILE_LINE_MAX + 1], *s;

126 -

127 - debug(D_BACKEND, "BACKEND: opening config file '%s'", filename);

128 -

129 - FILE *fp = fopen(filename, "r");

130 - if(!fp) {

131 - return 1;

132 - }

133 -

134 - while(fgets(buffer, CONFIG_FILE_LINE_MAX, fp) != NULL) {

135 - buffer[CONFIG_FILE_LINE_MAX] = '\0';

136 - line++;

137 -

138 - s = trim(buffer);

139 - if(!s || *s == '#') {

140 - debug(D_BACKEND, "BACKEND: ignoring line %d of file '%s', it is empty.", line, filename);

141 - continue;

142 - }

143 -

144 - char *name = s;

145 - char *value = strchr(s, '=');

146 - if(unlikely(!value)) {

147 - error("BACKEND: ignoring line %d ('%s') of file '%s', there is no = in it.", line, s, filename);

148 - continue;

149 - }

150 - *value = '\0';

151 - value++;

152 -

153 - name = trim(name);

154 - value = trim(value);

155 -

156 - if(unlikely(!name || *name == '#')) {

157 - error("BACKEND: ignoring line %d of file '%s', name is empty.", line, filename);

158 - continue;

159 - }

160 -

161 - if(!value)

162 - value = "";

163 - else

164 - value = strip_quotes(value);

165 -

166 - if(name[0] == 'u' && !strcmp(name, "uri")) {

167 - uri = strdupz(value);

168 - }

169 - else if(name[0] == 'd' && !strcmp(name, "database")) {

170 - database = strdupz(value);

171 - }

172 - else if(name[0] == 'c' && !strcmp(name, "collection")) {

173 - collection = strdupz(value);

174 - }

175 - }

176 -

177 - fclose(fp);

178 -

179 - if(unlikely(!collection || !*collection)) {

180 - error("BACKEND: collection name is a mandatory MongoDB parameter, but it is not configured");

181 - return 1;

182 - }

183 -

184 - *uri_p = uri;

185 - *database_p = database;

186 - *collection_p = collection;

187 -

188 - return 0;

189 -}

backends/mongodb/mongodb.conf deleted

-12

@@ -1,12 +0,0 @@

1 -# MongoDB backend configuration

2 -#

3 -# All options in this file are mandatory

4 -

5 -# URI

6 -uri =

7 -

8 -# database name

9 -database =

10 -

11 -# collection name

12 -collection =

backends/mongodb/mongodb.h deleted

-16

@@ -1,16 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_BACKEND_MONGODB_H

4 -#define NETDATA_BACKEND_MONGODB_H

5 -

6 -#include "backends/backends.h"

7 -

8 -extern int backends_mongodb_init(const char *uri_string, const char *database_string, const char *collection_string, const int32_t socket_timeout);

9 -

10 -extern int backends_mongodb_insert(char *data, size_t n_metrics);

11 -

12 -extern void backends_mongodb_cleanup();

13 -

14 -extern int read_mongodb_conf(const char *path, char **uri_p, char **database_p, char **collection_p);

15 -

16 -#endif //NETDATA_BACKEND_MONGODB_H

backends/nc-backend.sh deleted

-158

@@ -1,158 +0,0 @@

1 -#!/usr/bin/env bash

2 -

3 -# SPDX-License-Identifier: GPL-3.0-or-later

4 -

5 -# This is a simple backend database proxy, written in BASH, using the nc command.

6 -# Run the script without any parameters for help.

7 -

8 -MODE="${1}"

9 -MY_PORT="${2}"

10 -BACKEND_HOST="${3}"

11 -BACKEND_PORT="${4}"

12 -FILE="${NETDATA_NC_BACKEND_DIR-/tmp}/netdata-nc-backend-${MY_PORT}"

13 -

14 -log() {

15 - logger --stderr --id=$$ --tag "netdata-nc-backend" "${*}"

16 -}

17 -

18 -mync() {

19 - local ret

20 -

21 - log "Running: nc ${*}"

22 - nc "${@}"

23 - ret=$?

24 -

25 - log "nc stopped with return code ${ret}."

26 -

27 - return ${ret}

28 -}

29 -

30 -listen_save_replay_forever() {

31 - local file="${1}" port="${2}" real_backend_host="${3}" real_backend_port="${4}" ret delay=1 started ended

32 -

33 - while true

34 - do

35 - log "Starting nc to listen on port ${port} and save metrics to ${file}"

36 -

37 - started=$(date +%s)

38 - mync -l -p "${port}" | tee -a -p --output-error=exit "${file}"

39 - ended=$(date +%s)

40 -

41 - if [ -s "${file}" ]

42 - then

43 - if [ ! -z "${real_backend_host}" ] && [ ! -z "${real_backend_port}" ]

44 - then

45 - log "Attempting to send the metrics to the real backend at ${real_backend_host}:${real_backend_port}"

46 -

47 - mync "${real_backend_host}" "${real_backend_port}" <"${file}"

48 - ret=$?

49 -

50 - if [ ${ret} -eq 0 ]

51 - then

52 - log "Successfully sent the metrics to ${real_backend_host}:${real_backend_port}"

53 - mv "${file}" "${file}.old"

54 - touch "${file}"

55 - else

56 - log "Failed to send the metrics to ${real_backend_host}:${real_backend_port} (nc returned ${ret}) - appending more data to ${file}"

57 - fi

58 - else

59 - log "No backend configured - appending more data to ${file}"

60 - fi

61 - fi

62 -

63 - # prevent a CPU hungry infinite loop

64 - # if nc cannot listen to port

65 - if [ $((ended - started)) -lt 5 ]

66 - then

67 - log "nc has been stopped too fast."

68 - delay=30

69 - else

70 - delay=1

71 - fi

72 -

73 - log "Waiting ${delay} seconds before listening again for data."

74 - sleep ${delay}

75 - done

76 -}

77 -

78 -if [ "${MODE}" = "start" ]

79 - then

80 -

81 - # start the listener, in exclusive mode

82 - # only one can use the same file/port at a time

83 - {

84 - flock -n 9

85 - # shellcheck disable=SC2181

86 - if [ $? -ne 0 ]

87 - then

88 - log "Cannot get exclusive lock on file ${FILE}.lock - Am I running multiple times?"

89 - exit 2

90 - fi

91 -

92 - # save our PID to the lock file

93 - echo "$$" >"${FILE}.lock"

94 -

95 - listen_save_replay_forever "${FILE}" "${MY_PORT}" "${BACKEND_HOST}" "${BACKEND_PORT}"

96 - ret=$?

97 -

98 - log "listener exited."

99 - exit ${ret}

100 -

101 - } 9>>"${FILE}.lock"

102 -

103 - # we can only get here if ${FILE}.lock cannot be created

104 - log "Cannot create file ${FILE}."

105 - exit 3

106 -

107 -elif [ "${MODE}" = "stop" ]

108 - then

109 -

110 - {

111 - flock -n 9

112 - # shellcheck disable=SC2181

113 - if [ $? -ne 0 ]

114 - then

115 - pid=$(<"${FILE}".lock)

116 - log "Killing process ${pid}..."

117 - kill -TERM "-${pid}"

118 - exit 0

119 - fi

120 -

121 - log "File ${FILE}.lock has been locked by me but it shouldn't. Is a collector running?"

122 - exit 4

123 -

124 - } 9<"${FILE}.lock"

125 -

126 - log "File ${FILE}.lock does not exist. Is a collector running?"

127 - exit 5

128 -

129 -else

130 -

131 - cat <<EOF

132 -Usage:

133 -

134 - "${0}" start|stop PORT [BACKEND_HOST BACKEND_PORT]

135 -

136 - PORT The port this script will listen

137 - (configure netdata to use this as a second backend)

138 -

139 - BACKEND_HOST The real backend host

140 - BACKEND_PORT The real backend port

141 -

142 - This script can act as fallback backend for netdata.

143 - It will receive metrics from netdata, save them to

144 - ${FILE}

145 - and once netdata reconnects to the real-backend, this script

146 - will push all metrics collected to the real-backend too and

147 - wait for a failure to happen again.

148 -

149 - Only one netdata can connect to this script at a time.

150 - If you need fallback for multiple netdata, run this script

151 - multiple times with different ports.

152 -

153 - You can run me in the background with this:

154 -

155 - screen -d -m "${0}" start PORT [BACKEND_HOST BACKEND_PORT]

156 -EOF

157 - exit 1

158 -fi

backends/opentsdb/Makefile.am deleted

-4

@@ -1,4 +0,0 @@

1 -# SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -AUTOMAKE_OPTIONS = subdir-objects

4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in

backends/opentsdb/README.md deleted

-37

@@ -1,37 +0,0 @@

1 -<!--

2 -title: "OpenTSDB with HTTP"

3 -custom_edit_url: https://github.com/netdata/netdata/edit/master/backends/opentsdb/README.md

4 --->

5 -

6 -# OpenTSDB with HTTP

7 -

8 -Netdata can easily communicate with OpenTSDB using HTTP API. To enable this channel, set the following options in your

9 -`netdata.conf`:

10 -

11 -```conf

12 -[backend]

13 - type = opentsdb:http

14 - destination = localhost:4242

15 -```

16 -

17 -In this example, OpenTSDB is running with its default port, which is `4242`. If you run OpenTSDB on a different port,

18 -change the `destination = localhost:4242` line accordingly.

19 -

20 -## HTTPS

21 -

22 -As of [v1.16.0](https://github.com/netdata/netdata/releases/tag/v1.16.0), Netdata can send metrics to OpenTSDB using

23 -TLS/SSL. Unfortunately, OpenTDSB does not support encrypted connections, so you will have to configure a reverse proxy

24 -to enable HTTPS communication between Netdata and OpenTSDB. You can set up a reverse proxy with

25 -[Nginx](/docs/Running-behind-nginx.md).

26 -

27 -After your proxy is configured, make the following changes to `netdata.conf`:

28 -

29 -```conf

30 -[backend]

31 - type = opentsdb:https

32 - destination = localhost:8082

33 -```

34 -

35 -In this example, we used the port `8082` for our reverse proxy. If your reverse proxy listens on a different port,

36 -change the `destination = localhost:8082` line accordingly.

37 -

backends/opentsdb/opentsdb.c deleted

-205

@@ -1,205 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#define BACKENDS_INTERNALS

4 -#include "opentsdb.h"

5 -

6 -// ----------------------------------------------------------------------------

7 -// opentsdb backend

8 -

9 -int backends_format_dimension_collected_opentsdb_telnet(

10 - BUFFER *b // the buffer to write data to

11 - , const char *prefix // the prefix to use

12 - , RRDHOST *host // the host this chart comes from

13 - , const char *hostname // the hostname (to override host->hostname)

14 - , RRDSET *st // the chart

15 - , RRDDIM *rd // the dimension

16 - , time_t after // the start timestamp

17 - , time_t before // the end timestamp

18 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

19 -) {

20 - (void)host;

21 - (void)after;

22 - (void)before;

23 -

24 - char chart_name[RRD_ID_LENGTH_MAX + 1];

25 - char dimension_name[RRD_ID_LENGTH_MAX + 1];

26 - backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

27 - backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

28 -

29 - buffer_sprintf(

30 - b

31 - , "put %s.%s.%s %llu " COLLECTED_NUMBER_FORMAT " host=%s%s%s\n"

32 - , prefix

33 - , chart_name

34 - , dimension_name

35 - , (unsigned long long)rd->last_collected_time.tv_sec

36 - , rd->last_collected_value

37 - , hostname

38 - , (host->tags)?" ":""

39 - , (host->tags)?host->tags:""

40 - );

41 -

42 - return 1;

43 -}

44 -

45 -int backends_format_dimension_stored_opentsdb_telnet(

46 - BUFFER *b // the buffer to write data to

47 - , const char *prefix // the prefix to use

48 - , RRDHOST *host // the host this chart comes from

49 - , const char *hostname // the hostname (to override host->hostname)

50 - , RRDSET *st // the chart

51 - , RRDDIM *rd // the dimension

52 - , time_t after // the start timestamp

53 - , time_t before // the end timestamp

54 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

55 -) {

56 - (void)host;

57 -

58 - time_t first_t = after, last_t = before;

59 - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);

60 -

61 - char chart_name[RRD_ID_LENGTH_MAX + 1];

62 - char dimension_name[RRD_ID_LENGTH_MAX + 1];

63 - backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

64 - backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

65 -

66 - if(!isnan(value)) {

67 -

68 - buffer_sprintf(

69 - b

70 - , "put %s.%s.%s %llu " CALCULATED_NUMBER_FORMAT " host=%s%s%s\n"

71 - , prefix

72 - , chart_name

73 - , dimension_name

74 - , (unsigned long long) last_t

75 - , value

76 - , hostname

77 - , (host->tags)?" ":""

78 - , (host->tags)?host->tags:""

79 - );

80 -

81 - return 1;

82 - }

83 -

84 - return 0;

85 -}

86 -

87 -int process_opentsdb_response(BUFFER *b) {

88 - return discard_response(b, "opentsdb");

89 -}

90 -

91 -static inline void opentsdb_build_message(BUFFER *b, char *message, const char *hostname, int length) {

92 - buffer_sprintf(

93 - b

94 - , "POST /api/put HTTP/1.1\r\n"

95 - "Host: %s\r\n"

96 - "Content-Type: application/json\r\n"

97 - "Content-Length: %d\r\n"

98 - "\r\n"

99 - "%s"

100 - , hostname

101 - , length

102 - , message

103 - );

104 -}

105 -

106 -int backends_format_dimension_collected_opentsdb_http(

107 - BUFFER *b // the buffer to write data to

108 - , const char *prefix // the prefix to use

109 - , RRDHOST *host // the host this chart comes from

110 - , const char *hostname // the hostname (to override host->hostname)

111 - , RRDSET *st // the chart

112 - , RRDDIM *rd // the dimension

113 - , time_t after // the start timestamp

114 - , time_t before // the end timestamp

115 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

116 -) {

117 - (void)host;

118 - (void)after;

119 - (void)before;

120 -

121 - char message[1024];

122 - char chart_name[RRD_ID_LENGTH_MAX + 1];

123 - char dimension_name[RRD_ID_LENGTH_MAX + 1];

124 - backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

125 - backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

126 -

127 - int length = snprintfz(message

128 - , sizeof(message)

129 - , "{"

130 - " \"metric\": \"%s.%s.%s\","

131 - " \"timestamp\": %llu,"

132 - " \"value\": "COLLECTED_NUMBER_FORMAT ","

133 - " \"tags\": {"

134 - " \"host\": \"%s%s%s\""

135 - " }"

136 - "}"

137 - , prefix

138 - , chart_name

139 - , dimension_name

140 - , (unsigned long long)rd->last_collected_time.tv_sec

141 - , rd->last_collected_value

142 - , hostname

143 - , (host->tags)?" ":""

144 - , (host->tags)?host->tags:""

145 - );

146 -

147 - if(length > 0) {

148 - opentsdb_build_message(b, message, hostname, length);

149 - }

150 -

151 - return 1;

152 -}

153 -

154 -int backends_format_dimension_stored_opentsdb_http(

155 - BUFFER *b // the buffer to write data to

156 - , const char *prefix // the prefix to use

157 - , RRDHOST *host // the host this chart comes from

158 - , const char *hostname // the hostname (to override host->hostname)

159 - , RRDSET *st // the chart

160 - , RRDDIM *rd // the dimension

161 - , time_t after // the start timestamp

162 - , time_t before // the end timestamp

163 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

164 -) {

165 - (void)host;

166 -

167 - time_t first_t = after, last_t = before;

168 - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);

169 -

170 - if(!isnan(value)) {

171 - char chart_name[RRD_ID_LENGTH_MAX + 1];

172 - char dimension_name[RRD_ID_LENGTH_MAX + 1];

173 - backend_name_copy(chart_name, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, RRD_ID_LENGTH_MAX);

174 - backend_name_copy(dimension_name, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name)?rd->name:rd->id, RRD_ID_LENGTH_MAX);

175 -

176 - char message[1024];

177 - int length = snprintfz(message

178 - , sizeof(message)

179 - , "{"

180 - " \"metric\": \"%s.%s.%s\","

181 - " \"timestamp\": %llu,"

182 - " \"value\": "CALCULATED_NUMBER_FORMAT ","

183 - " \"tags\": {"

184 - " \"host\": \"%s%s%s\""

185 - " }"

186 - "}"

187 - , prefix

188 - , chart_name

189 - , dimension_name

190 - , (unsigned long long)last_t

191 - , value

192 - , hostname

193 - , (host->tags)?" ":""

194 - , (host->tags)?host->tags:""

195 - );

196 -

197 - if(length > 0) {

198 - opentsdb_build_message(b, message, hostname, length);

199 - }

200 -

201 - return 1;

202 - }

203 -

204 - return 0;

205 -}

backends/opentsdb/opentsdb.h deleted

-58

@@ -1,58 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_BACKEND_OPENTSDB_H

4 -#define NETDATA_BACKEND_OPENTSDB_H

5 -

6 -#include "backends/backends.h"

7 -

8 -extern int backends_format_dimension_collected_opentsdb_telnet(

9 - BUFFER *b // the buffer to write data to

10 - , const char *prefix // the prefix to use

11 - , RRDHOST *host // the host this chart comes from

12 - , const char *hostname // the hostname (to override host->hostname)

13 - , RRDSET *st // the chart

14 - , RRDDIM *rd // the dimension

15 - , time_t after // the start timestamp

16 - , time_t before // the end timestamp

17 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

18 -);

19 -

20 -extern int backends_format_dimension_stored_opentsdb_telnet(

21 - BUFFER *b // the buffer to write data to

22 - , const char *prefix // the prefix to use

23 - , RRDHOST *host // the host this chart comes from

24 - , const char *hostname // the hostname (to override host->hostname)

25 - , RRDSET *st // the chart

26 - , RRDDIM *rd // the dimension

27 - , time_t after // the start timestamp

28 - , time_t before // the end timestamp

29 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

30 -);

31 -

32 -extern int process_opentsdb_response(BUFFER *b);

33 -

34 -int backends_format_dimension_collected_opentsdb_http(

35 - BUFFER *b // the buffer to write data to

36 - , const char *prefix // the prefix to use

37 - , RRDHOST *host // the host this chart comes from

38 - , const char *hostname // the hostname (to override host->hostname)

39 - , RRDSET *st // the chart

40 - , RRDDIM *rd // the dimension

41 - , time_t after // the start timestamp

42 - , time_t before // the end timestamp

43 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

44 -);

45 -

46 -int backends_format_dimension_stored_opentsdb_http(

47 - BUFFER *b // the buffer to write data to

48 - , const char *prefix // the prefix to use

49 - , RRDHOST *host // the host this chart comes from

50 - , const char *hostname // the hostname (to override host->hostname)

51 - , RRDSET *st // the chart

52 - , RRDDIM *rd // the dimension

53 - , time_t after // the start timestamp

54 - , time_t before // the end timestamp

55 - , BACKEND_OPTIONS backend_options // BACKEND_SOURCE_* bitmap

56 -);

57 -

58 -#endif //NETDATA_BACKEND_OPENTSDB_H

backends/prometheus/Makefile.am deleted

-12

@@ -1,12 +0,0 @@

1 -# SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -AUTOMAKE_OPTIONS = subdir-objects

4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in

5 -

6 -SUBDIRS = \

7 - remote_write \

8 - $(NULL)

9 -

10 -dist_noinst_DATA = \

11 - README.md \

12 - $(NULL)

backends/prometheus/README.md deleted

-457

@@ -1,457 +0,0 @@

1 -<!--

2 -title: "Using Netdata with Prometheus"

3 -custom_edit_url: https://github.com/netdata/netdata/edit/master/backends/prometheus/README.md

4 --->

5 -

6 -# Using Netdata with Prometheus

7 -

8 -> IMPORTANT: the format Netdata sends metrics to prometheus has changed since Netdata v1.7. The new prometheus backend

9 -> for Netdata supports a lot more features and is aligned to the development of the rest of the Netdata backends.

10 -

11 -Prometheus is a distributed monitoring system which offers a very simple setup along with a robust data model. Recently

12 -Netdata added support for Prometheus. I'm going to quickly show you how to install both Netdata and prometheus on the

13 -same server. We can then use grafana pointed at Prometheus to obtain long term metrics Netdata offers. I'm assuming we

14 -are starting at a fresh ubuntu shell (whether you'd like to follow along in a VM or a cloud instance is up to you).

15 -

16 -## Installing Netdata and prometheus

17 -

18 -### Installing Netdata

19 -

20 -There are number of ways to install Netdata according to [Installation](/packaging/installer/README.md). The suggested way

21 -of installing the latest Netdata and keep it upgrade automatically. Using one line installation:

22 -

23 -```sh

24 -bash <(curl -Ss https://my-netdata.io/kickstart.sh)

25 -```

26 -

27 -At this point we should have Netdata listening on port 19999. Attempt to take your browser here:

28 -

29 -```sh

30 -http://your.netdata.ip:19999

31 -```

32 -

33 -_(replace `your.netdata.ip` with the IP or hostname of the server running Netdata)_

34 -

35 -### Installing Prometheus

36 -

37 -In order to install prometheus we are going to introduce our own systemd startup script along with an example of

38 -prometheus.yaml configuration. Prometheus needs to be pointed to your server at a specific target url for it to scrape

39 -Netdata's api. Prometheus is always a pull model meaning Netdata is the passive client within this architecture.

40 -Prometheus always initiates the connection with Netdata.

41 -

42 -#### Download Prometheus

43 -

44 -```sh

45 -cd /tmp && curl -s https://api.github.com/repos/prometheus/prometheus/releases/latest \

46 -| grep "browser_download_url.*linux-amd64.tar.gz" \

47 -| cut -d '"' -f 4 \

48 -| wget -qi -

49 -```

50 -

51 -#### Create prometheus system user

52 -

53 -```sh

54 -sudo useradd -r prometheus

55 -```

56 -

57 -#### Create prometheus directory

58 -

59 -```sh

60 -sudo mkdir /opt/prometheus

61 -sudo chown prometheus:prometheus /opt/prometheus

62 -```

63 -

64 -#### Untar prometheus directory

65 -

66 -```sh

67 -sudo tar -xvf /tmp/prometheus-*linux-amd64.tar.gz -C /opt/prometheus --strip=1

68 -```

69 -

70 -#### Install prometheus.yml

71 -

72 -We will use the following `prometheus.yml` file. Save it at `/opt/prometheus/prometheus.yml`.

73 -

74 -Make sure to replace `your.netdata.ip` with the IP or hostname of the host running Netdata.

75 -

76 -```yaml

77 -# my global config

78 -global:

79 - scrape_interval: 5s # Set the scrape interval to every 5 seconds. Default is every 1 minute.

80 - evaluation_interval: 5s # Evaluate rules every 5 seconds. The default is every 1 minute.

81 - # scrape_timeout is set to the global default (10s).

82 -

83 - # Attach these labels to any time series or alerts when communicating with

84 - # external systems (federation, remote storage, Alertmanager).

85 - external_labels:

86 - monitor: 'codelab-monitor'

87 -

88 -# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.

89 -rule_files:

90 - # - "first.rules"

91 - # - "second.rules"

92 -

93 -# A scrape configuration containing exactly one endpoint to scrape:

94 -# Here it's Prometheus itself.

95 -scrape_configs:

96 - # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.

97 - - job_name: 'prometheus'

98 -

99 - # metrics_path defaults to '/metrics'

100 - # scheme defaults to 'http'.

101 -

102 - static_configs:

103 - - targets: ['0.0.0.0:9090']

104 -

105 - - job_name: 'netdata-scrape'

106 -

107 - metrics_path: '/api/v1/allmetrics'

108 - params:

109 - # format: prometheus | prometheus_all_hosts

110 - # You can use `prometheus_all_hosts` if you want Prometheus to set the `instance` to your hostname instead of IP

111 - format: [prometheus]

112 - #

113 - # source: as-collected | raw | average | sum | volume

114 - # default is: average

115 - #source: [as-collected]

116 - #

117 - # server name for this prometheus - the default is the client IP

118 - # for Netdata to uniquely identify it

119 - #server: ['prometheus1']

120 - honor_labels: true

121 -

122 - static_configs:

123 - - targets: ['{your.netdata.ip}:19999']

124 -```

125 -

126 -#### Install nodes.yml

127 -

128 -The following is completely optional, it will enable Prometheus to generate alerts from some Netdata sources. Tweak the

129 -values to your own needs. We will use the following `nodes.yml` file below. Save it at `/opt/prometheus/nodes.yml`, and

130 -add a _- "nodes.yml"_ entry under the _rule_files:_ section in the example prometheus.yml file above.

131 -

132 -```yaml

133 -groups:

134 - - name: nodes

135 -

136 - rules:

137 - - alert: node_high_cpu_usage_70

138 - expr: sum(sum_over_time(netdata_system_cpu_percentage_average{dimension=~"(user|system|softirq|irq|guest)"}[10m])) by (job) / sum(count_over_time(netdata_system_cpu_percentage_average{dimension="idle"}[10m])) by (job) > 70

139 - for: 1m

140 - annotations:

141 - description: '{{ $labels.job }} on ''{{ $labels.job }}'' CPU usage is at {{ humanize $value }}%.'

142 - summary: CPU alert for container node '{{ $labels.job }}'

143 -

144 - - alert: node_high_memory_usage_70

145 - expr: 100 / sum(netdata_system_ram_MB_average) by (job)

146 - * sum(netdata_system_ram_MB_average{dimension=~"free|cached"}) by (job) < 30

147 - for: 1m

148 - annotations:

149 - description: '{{ $labels.job }} memory usage is {{ humanize $value}}%.'

150 - summary: Memory alert for container node '{{ $labels.job }}'

151 -

152 - - alert: node_low_root_filesystem_space_20

153 - expr: 100 / sum(netdata_disk_space_GB_average{family="/"}) by (job)

154 - * sum(netdata_disk_space_GB_average{family="/",dimension=~"avail|cached"}) by (job) < 20

155 - for: 1m

156 - annotations:

157 - description: '{{ $labels.job }} root filesystem space is {{ humanize $value}}%.'

158 - summary: Root filesystem alert for container node '{{ $labels.job }}'

159 -

160 - - alert: node_root_filesystem_fill_rate_6h

161 - expr: predict_linear(netdata_disk_space_GB_average{family="/",dimension=~"avail|cached"}[1h], 6 * 3600) < 0

162 - for: 1h

163 - labels:

164 - severity: critical

165 - annotations:

166 - description: Container node {{ $labels.job }} root filesystem is going to fill up in 6h.

167 - summary: Disk fill alert for Swarm node '{{ $labels.job }}'

168 -```

169 -

170 -#### Install prometheus.service

171 -

172 -Save this service file as `/etc/systemd/system/prometheus.service`:

173 -

174 -```sh

175 -[Unit]

176 -Description=Prometheus Server

177 -AssertPathExists=/opt/prometheus

178 -

179 -[Service]

180 -Type=simple

181 -WorkingDirectory=/opt/prometheus

182 -User=prometheus

183 -Group=prometheus

184 -ExecStart=/opt/prometheus/prometheus --config.file=/opt/prometheus/prometheus.yml --log.level=info

185 -ExecReload=/bin/kill -SIGHUP $MAINPID

186 -ExecStop=/bin/kill -SIGINT $MAINPID

187 -

188 -[Install]

189 -WantedBy=multi-user.target

190 -```

191 -

192 -##### Start Prometheus

193 -

194 -```sh

195 -sudo systemctl start prometheus

196 -sudo systemctl enable prometheus

197 -```

198 -

199 -Prometheus should now start and listen on port 9090. Attempt to head there with your browser.

200 -

201 -If everything is working correctly when you fetch `http://your.prometheus.ip:9090` you will see a 'Status' tab. Click

202 -this and click on 'targets' We should see the Netdata host as a scraped target.

203 -

204 ----

205 -

206 -## Netdata support for prometheus

207 -

208 -> IMPORTANT: the format Netdata sends metrics to prometheus has changed since Netdata v1.6. The new format allows easier

209 -> queries for metrics and supports both `as collected` and normalized metrics.

210 -

211 -Before explaining the changes, we have to understand the key differences between Netdata and prometheus.

212 -

213 -### understanding Netdata metrics

214 -

215 -#### charts

216 -

217 -Each chart in Netdata has several properties (common to all its metrics):

218 -

219 -- `chart_id` - uniquely identifies a chart.

220 -

221 -- `chart_name` - a more human friendly name for `chart_id`, also unique.

222 -

223 -- `context` - this is the template of the chart. All disk I/O charts have the same context, all mysql requests charts

224 - have the same context, etc. This is used for alarm templates to match all the charts they should be attached to.

225 -

226 -- `family` groups a set of charts together. It is used as the submenu of the dashboard.

227 -

228 -- `units` is the units for all the metrics attached to the chart.

229 -

230 -#### dimensions

231 -

232 -Then each Netdata chart contains metrics called `dimensions`. All the dimensions of a chart have the same units of

233 -measurement, and are contextually in the same category (ie. the metrics for disk bandwidth are `read` and `write` and

234 -they are both in the same chart).

235 -

236 -### Netdata data source

237 -

238 -Netdata can send metrics to prometheus from 3 data sources:

239 -

240 -- `as collected` or `raw` - this data source sends the metrics to prometheus as they are collected. No conversion is

241 - done by Netdata. The latest value for each metric is just given to prometheus. This is the most preferred method by

242 - prometheus, but it is also the harder to work with. To work with this data source, you will need to understand how

243 - to get meaningful values out of them.

244 -

245 - The format of the metrics is: `CONTEXT{chart="CHART",family="FAMILY",dimension="DIMENSION"}`.

246 -

247 - If the metric is a counter (`incremental` in Netdata lingo), `_total` is appended the context.

248 -

249 - Unlike prometheus, Netdata allows each dimension of a chart to have a different algorithm and conversion constants

250 - (`multiplier` and `divisor`). In this case, that the dimensions of a charts are heterogeneous, Netdata will use this

251 - format: `CONTEXT_DIMENSION{chart="CHART",family="FAMILY"}`

252 -

253 -- `average` - this data source uses the Netdata database to send the metrics to prometheus as they are presented on

254 - the Netdata dashboard. So, all the metrics are sent as gauges, at the units they are presented in the Netdata

255 - dashboard charts. This is the easiest to work with.

256 -

257 - The format of the metrics is: `CONTEXT_UNITS_average{chart="CHART",family="FAMILY",dimension="DIMENSION"}`.

258 -

259 - When this source is used, Netdata keeps track of the last access time for each prometheus server fetching the

260 - metrics. This last access time is used at the subsequent queries of the same prometheus server to identify the

261 - time-frame the `average` will be calculated.

262 -

263 - So, no matter how frequently prometheus scrapes Netdata, it will get all the database data.

264 - To identify each prometheus server, Netdata uses by default the IP of the client fetching the metrics.

265 -

266 - If there are multiple prometheus servers fetching data from the same Netdata, using the same IP, each prometheus

267 - server can append `server=NAME` to the URL. Netdata will use this `NAME` to uniquely identify the prometheus server.

268 -

269 -- `sum` or `volume`, is like `average` but instead of averaging the values, it sums them.

270 -

271 - The format of the metrics is: `CONTEXT_UNITS_sum{chart="CHART",family="FAMILY",dimension="DIMENSION"}`. All the

272 - other operations are the same with `average`.

273 -

274 - To change the data source to `sum` or `as-collected` you need to provide the `source` parameter in the request URL.

275 - e.g.: `http://your.netdata.ip:19999/api/v1/allmetrics?format=prometheus&help=yes&source=as-collected`

276 -

277 - Keep in mind that early versions of Netdata were sending the metrics as: `CHART_DIMENSION{}`.

278 -

279 -### Querying Metrics

280 -

281 -Fetch with your web browser this URL:

282 -

283 -`http://your.netdata.ip:19999/api/v1/allmetrics?format=prometheus&help=yes`

284 -

285 -_(replace `your.netdata.ip` with the ip or hostname of your Netdata server)_

286 -

287 -Netdata will respond with all the metrics it sends to prometheus.

288 -

289 -If you search that page for `"system.cpu"` you will find all the metrics Netdata is exporting to prometheus for this

290 -chart. `system.cpu` is the chart name on the Netdata dashboard (on the Netdata dashboard all charts have a text heading

291 -such as : `Total CPU utilization (system.cpu)`. What we are interested here in the chart name: `system.cpu`).

292 -

293 -Searching for `"system.cpu"` reveals:

294 -

295 -```sh

296 -# COMMENT homogeneous chart "system.cpu", context "system.cpu", family "cpu", units "percentage"

297 -# COMMENT netdata_system_cpu_percentage_average: dimension "guest_nice", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive

298 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="guest_nice"} 0.0000000 1500066662000

299 -# COMMENT netdata_system_cpu_percentage_average: dimension "guest", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive

300 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="guest"} 1.7837326 1500066662000

301 -# COMMENT netdata_system_cpu_percentage_average: dimension "steal", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive

302 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="steal"} 0.0000000 1500066662000

303 -# COMMENT netdata_system_cpu_percentage_average: dimension "softirq", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive

304 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="softirq"} 0.5275442 1500066662000

305 -# COMMENT netdata_system_cpu_percentage_average: dimension "irq", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive

306 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="irq"} 0.2260836 1500066662000

307 -# COMMENT netdata_system_cpu_percentage_average: dimension "user", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive

308 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="user"} 2.3362762 1500066662000

309 -# COMMENT netdata_system_cpu_percentage_average: dimension "system", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive

310 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="system"} 1.7961062 1500066662000

311 -# COMMENT netdata_system_cpu_percentage_average: dimension "nice", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive

312 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="nice"} 0.0000000 1500066662000

313 -# COMMENT netdata_system_cpu_percentage_average: dimension "iowait", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive

314 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="iowait"} 0.9671802 1500066662000

315 -# COMMENT netdata_system_cpu_percentage_average: dimension "idle", value is percentage, gauge, dt 1500066653 to 1500066662 inclusive

316 -netdata_system_cpu_percentage_average{chart="system.cpu",family="cpu",dimension="idle"} 92.3630770 1500066662000

317 -```

318 -

319 -_(Netdata response for `system.cpu` with source=`average`)_

320 -

321 -In `average` or `sum` data sources, all values are normalized and are reported to prometheus as gauges. Now, use the

322 -'expression' text form in prometheus. Begin to type the metrics we are looking for: `netdata_system_cpu`. You should see

323 -that the text form begins to auto-fill as prometheus knows about this metric.

324 -

325 -If the data source was `as collected`, the response would be:

326 -

327 -```sh

328 -# COMMENT homogeneous chart "system.cpu", context "system.cpu", family "cpu", units "percentage"

329 -# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "guest_nice", value * 1 / 1 delta gives percentage (counter)

330 -netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="guest_nice"} 0 1500066716438

331 -# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "guest", value * 1 / 1 delta gives percentage (counter)

332 -netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="guest"} 63945 1500066716438

333 -# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "steal", value * 1 / 1 delta gives percentage (counter)

334 -netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="steal"} 0 1500066716438

335 -# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "softirq", value * 1 / 1 delta gives percentage (counter)

336 -netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="softirq"} 8295 1500066716438

337 -# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "irq", value * 1 / 1 delta gives percentage (counter)

338 -netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="irq"} 4079 1500066716438

339 -# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "user", value * 1 / 1 delta gives percentage (counter)

340 -netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="user"} 116488 1500066716438

341 -# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "system", value * 1 / 1 delta gives percentage (counter)

342 -netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="system"} 35084 1500066716438

343 -# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "nice", value * 1 / 1 delta gives percentage (counter)

344 -netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="nice"} 505 1500066716438

345 -# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "iowait", value * 1 / 1 delta gives percentage (counter)

346 -netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="iowait"} 23314 1500066716438

347 -# COMMENT netdata_system_cpu_total: chart "system.cpu", context "system.cpu", family "cpu", dimension "idle", value * 1 / 1 delta gives percentage (counter)

348 -netdata_system_cpu_total{chart="system.cpu",family="cpu",dimension="idle"} 918470 1500066716438

349 -```

350 -

351 -_(Netdata response for `system.cpu` with source=`as-collected`)_

352 -

353 -For more information check prometheus documentation.

354 -

355 -### Streaming data from upstream hosts

356 -

357 -The `format=prometheus` parameter only exports the host's Netdata metrics. If you are using the parent-child

358 -functionality of Netdata this ignores any upstream hosts - so you should consider using the below in your

359 -**prometheus.yml**:

360 -

361 -```yaml

362 - metrics_path: '/api/v1/allmetrics'

363 - params:

364 - format: [prometheus_all_hosts]

365 - honor_labels: true

366 -```

367 -

368 -This will report all upstream host data, and `honor_labels` will make Prometheus take note of the instance names

369 -provided.

370 -

371 -### Timestamps

372 -

373 -To pass the metrics through prometheus pushgateway, Netdata supports the option `&timestamps=no` to send the metrics

374 -without timestamps.

375 -

376 -## Netdata host variables

377 -

378 -Netdata collects various system configuration metrics, like the max number of TCP sockets supported, the max number of

379 -files allowed system-wide, various IPC sizes, etc. These metrics are not exposed to prometheus by default.

380 -

381 -To expose them, append `variables=yes` to the Netdata URL.

382 -

383 -### TYPE and HELP

384 -

385 -To save bandwidth, and because prometheus does not use them anyway, `# TYPE` and `# HELP` lines are suppressed. If

386 -wanted they can be re-enabled via `types=yes` and `help=yes`, e.g.

387 -`/api/v1/allmetrics?format=prometheus&types=yes&help=yes`

388 -

389 -Note that if enabled, the `# TYPE` and `# HELP` lines are repeated for every occurrence of a metric, which goes against the Prometheus documentation's [specification for these lines](https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#comments-help-text-and-type-information).

390 -

391 -### Names and IDs

392 -

393 -Netdata supports names and IDs for charts and dimensions. Usually IDs are unique identifiers as read by the system and

394 -names are human friendly labels (also unique).

395 -

396 -Most charts and metrics have the same ID and name, but in several cases they are different: disks with device-mapper,

397 -interrupts, QoS classes, statsd synthetic charts, etc.

398 -

399 -The default is controlled in `netdata.conf`:

400 -

401 -```conf

402 -[backend]

403 - send names instead of ids = yes | no

404 -```

405 -

406 -You can overwrite it from prometheus, by appending to the URL:

407 -

408 -- `&names=no` to get IDs (the old behaviour)

409 -- `&names=yes` to get names

410 -

411 -### Filtering metrics sent to prometheus

412 -

413 -Netdata can filter the metrics it sends to prometheus with this setting:

414 -

415 -```conf

416 -[backend]

417 - send charts matching = *

418 -```

419 -

420 -This settings accepts a space separated list of patterns to match the **charts** to be sent to prometheus. Each pattern

421 -can use `*` as wildcard, any number of times (e.g `*a*b*c*` is valid). Patterns starting with `!` give a negative match

422 -(e.g `!*.bad users.* groups.*` will send all the users and groups except `bad` user and `bad` group). The order is

423 -important: the first match (positive or negative) left to right, is used.

424 -

425 -### Changing the prefix of Netdata metrics

426 -

427 -Netdata sends all metrics prefixed with `netdata_`. You can change this in `netdata.conf`, like this:

428 -

429 -```conf

430 -[backend]

431 - prefix = netdata

432 -```

433 -

434 -It can also be changed from the URL, by appending `&prefix=netdata`.

435 -

436 -### Metric Units

437 -

438 -The default source `average` adds the unit of measurement to the name of each metric (e.g. `_KiB_persec`). To hide the

439 -units and get the same metric names as with the other sources, append to the URL `&hideunits=yes`.

440 -

441 -The units were standardized in v1.12, with the effect of changing the metric names. To get the metric names as they were

442 -before v1.12, append to the URL `&oldunits=yes`

443 -

444 -### Accuracy of `average` and `sum` data sources

445 -

446 -When the data source is set to `average` or `sum`, Netdata remembers the last access of each client accessing prometheus

447 -metrics and uses this last access time to respond with the `average` or `sum` of all the entries in the database since

448 -that. This means that prometheus servers are not losing data when they access Netdata with data source = `average` or

449 -`sum`.

450 -

451 -To uniquely identify each prometheus server, Netdata uses the IP of the client accessing the metrics. If however the IP

452 -is not good enough for identifying a single prometheus server (e.g. when prometheus servers are accessing Netdata

453 -through a web proxy, or when multiple prometheus servers are NATed to a single IP), each prometheus may append

454 -`&server=NAME` to the URL. This `NAME` is used by Netdata to uniquely identify each prometheus server and keep track of

455 -its last access time.

456 -

457 -

backends/prometheus/backend_prometheus.c deleted

-797

@@ -1,797 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#define BACKENDS_INTERNALS

4 -#include "backend_prometheus.h"

5 -

6 -// ----------------------------------------------------------------------------

7 -// PROMETHEUS

8 -// /api/v1/allmetrics?format=prometheus and /api/v1/allmetrics?format=prometheus_all_hosts

9 -

10 -static struct prometheus_server {

11 - const char *server;

12 - uint32_t hash;

13 - RRDHOST *host;

14 - time_t last_access;

15 - struct prometheus_server *next;

16 -} *prometheus_server_root = NULL;

17 -

18 -static inline time_t prometheus_server_last_access(const char *server, RRDHOST *host, time_t now) {

19 - static netdata_mutex_t prometheus_server_root_mutex = NETDATA_MUTEX_INITIALIZER;

20 -

21 - uint32_t hash = simple_hash(server);

22 -

23 - netdata_mutex_lock(&prometheus_server_root_mutex);

24 -

25 - struct prometheus_server *ps;

26 - for(ps = prometheus_server_root; ps ;ps = ps->next) {

27 - if (host == ps->host && hash == ps->hash && !strcmp(server, ps->server)) {

28 - time_t last = ps->last_access;

29 - ps->last_access = now;

30 - netdata_mutex_unlock(&prometheus_server_root_mutex);

31 - return last;

32 - }

33 - }

34 -

35 - ps = callocz(1, sizeof(struct prometheus_server));

36 - ps->server = strdupz(server);

37 - ps->hash = hash;

38 - ps->host = host;

39 - ps->last_access = now;

40 - ps->next = prometheus_server_root;

41 - prometheus_server_root = ps;

42 -

43 - netdata_mutex_unlock(&prometheus_server_root_mutex);

44 - return 0;

45 -}

46 -

47 -static inline size_t backends_prometheus_name_copy(char *d, const char *s, size_t usable) {

48 - size_t n;

49 -

50 - for(n = 0; *s && n < usable ; d++, s++, n++) {

51 - register char c = *s;

52 -

53 - if(!isalnum(c)) *d = '_';

54 - else *d = c;

55 - }

56 - *d = '\0';

57 -

58 - return n;

59 -}

60 -

61 -static inline size_t backends_prometheus_label_copy(char *d, const char *s, size_t usable) {

62 - size_t n;

63 -

64 - // make sure we can escape one character without overflowing the buffer

65 - usable--;

66 -

67 - for(n = 0; *s && n < usable ; d++, s++, n++) {

68 - register char c = *s;

69 -

70 - if(unlikely(c == '"' || c == '\\' || c == '\n')) {

71 - *d++ = '\\';

72 - n++;

73 - }

74 - *d = c;

75 - }

76 - *d = '\0';

77 -

78 - return n;

79 -}

80 -

81 -static inline char *backends_prometheus_units_copy(char *d, const char *s, size_t usable, int showoldunits) {

82 - const char *sorig = s;

83 - char *ret = d;

84 - size_t n;

85 -

86 - // Fix for issue 5227

87 - if (unlikely(showoldunits)) {

88 - static struct {

89 - const char *newunit;

90 - uint32_t hash;

91 - const char *oldunit;

92 - } units[] = {

93 - {"KiB/s", 0, "kilobytes/s"}

94 - , {"MiB/s", 0, "MB/s"}

95 - , {"GiB/s", 0, "GB/s"}

96 - , {"KiB" , 0, "KB"}

97 - , {"MiB" , 0, "MB"}

98 - , {"GiB" , 0, "GB"}

99 - , {"inodes" , 0, "Inodes"}

100 - , {"percentage" , 0, "percent"}

101 - , {"faults/s" , 0, "page faults/s"}

102 - , {"KiB/operation", 0, "kilobytes per operation"}

103 - , {"milliseconds/operation", 0, "ms per operation"}

104 - , {NULL, 0, NULL}

105 - };

106 - static int initialized = 0;

107 - int i;

108 -

109 - if(unlikely(!initialized)) {

110 - for (i = 0; units[i].newunit; i++)

111 - units[i].hash = simple_hash(units[i].newunit);

112 - initialized = 1;

113 - }

114 -

115 - uint32_t hash = simple_hash(s);

116 - for(i = 0; units[i].newunit ; i++) {

117 - if(unlikely(hash == units[i].hash && !strcmp(s, units[i].newunit))) {

118 - // info("matched extension for filename '%s': '%s'", filename, last_dot);

119 - s=units[i].oldunit;

120 - sorig = s;

121 - break;

122 - }

123 - }

124 - }

125 - *d++ = '_';

126 - for(n = 1; *s && n < usable ; d++, s++, n++) {

127 - register char c = *s;

128 -

129 - if(!isalnum(c)) *d = '_';

130 - else *d = c;

131 - }

132 -

133 - if(n == 2 && sorig[0] == '%') {

134 - n = 0;

135 - d = ret;

136 - s = "_percent";

137 - for( ; *s && n < usable ; n++) *d++ = *s++;

138 - }

139 - else if(n > 3 && sorig[n-3] == '/' && sorig[n-2] == 's') {

140 - n = n - 2;

141 - d -= 2;

142 - s = "_persec";

143 - for( ; *s && n < usable ; n++) *d++ = *s++;

144 - }

145 -

146 - *d = '\0';

147 -

148 - return ret;

149 -}

150 -

151 -

152 -#define PROMETHEUS_ELEMENT_MAX 256

153 -#define PROMETHEUS_LABELS_MAX 1024

154 -#define PROMETHEUS_VARIABLE_MAX 256

155 -

156 -#define PROMETHEUS_LABELS_MAX_NUMBER 128

157 -

158 -struct host_variables_callback_options {

159 - RRDHOST *host;

160 - BUFFER *wb;

161 - BACKEND_OPTIONS backend_options;

162 - BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options;

163 - const char *prefix;

164 - const char *labels;

165 - time_t now;

166 - int host_header_printed;

167 - char name[PROMETHEUS_VARIABLE_MAX+1];

168 -};

169 -

170 -static int print_host_variables(RRDVAR *rv, void *data) {

171 - struct host_variables_callback_options *opts = data;

172 -

173 - if(rv->options & (RRDVAR_OPTION_CUSTOM_HOST_VAR|RRDVAR_OPTION_CUSTOM_CHART_VAR)) {

174 - if(!opts->host_header_printed) {

175 - opts->host_header_printed = 1;

176 -

177 - if(opts->output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP) {

178 - buffer_sprintf(opts->wb, "\n# COMMENT global host and chart variables\n");

179 - }

180 - }

181 -

182 - calculated_number value = rrdvar2number(rv);

183 - if(isnan(value) || isinf(value)) {

184 - if(opts->output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP)

185 - buffer_sprintf(opts->wb, "# COMMENT variable \"%s\" is %s. Skipped.\n", rv->name, (isnan(value))?"NAN":"INF");

186 -

187 - return 0;

188 - }

189 -

190 - char *label_pre = "";

191 - char *label_post = "";

192 - if(opts->labels && *opts->labels) {

193 - label_pre = "{";

194 - label_post = "}";

195 - }

196 -

197 - backends_prometheus_name_copy(opts->name, rv->name, sizeof(opts->name));

198 -

199 - if(opts->output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)

200 - buffer_sprintf(opts->wb

201 - , "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT " %llu\n"

202 - , opts->prefix

203 - , opts->name

204 - , label_pre

205 - , opts->labels

206 - , label_post

207 - , value

208 - , opts->now * 1000ULL

209 - );

210 - else

211 - buffer_sprintf(opts->wb, "%s_%s%s%s%s " CALCULATED_NUMBER_FORMAT "\n"

212 - , opts->prefix

213 - , opts->name

214 - , label_pre

215 - , opts->labels

216 - , label_post

217 - , value

218 - );

219 -

220 - return 1;

221 - }

222 -

223 - return 0;

224 -}

225 -

226 -static void rrd_stats_api_v1_charts_allmetrics_prometheus(RRDHOST *host, BUFFER *wb, const char *prefix, BACKEND_OPTIONS backend_options, time_t after, time_t before, int allhosts, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options) {

227 - rrdhost_rdlock(host);

228 -

229 - char hostname[PROMETHEUS_ELEMENT_MAX + 1];

230 - backends_prometheus_label_copy(hostname, host->hostname, PROMETHEUS_ELEMENT_MAX);

231 -

232 - char labels[PROMETHEUS_LABELS_MAX + 1] = "";

233 - if(allhosts) {

234 - if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)

235 - buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);

236 - else

237 - buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", hostname, host->program_name, host->program_version);

238 -

239 - if(host->tags && *(host->tags)) {

240 - if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS) {

241 - buffer_sprintf(wb, "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS);

242 -

243 - // deprecated, exists only for compatibility with older queries

244 - buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n", hostname, host->tags, now_realtime_usec() / USEC_PER_MS);

245 - }

246 - else {

247 - buffer_sprintf(wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, host->tags);

248 -

249 - // deprecated, exists only for compatibility with older queries

250 - buffer_sprintf(wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, host->tags);

251 - }

252 -

253 - }

254 -

255 - snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);

256 - }

257 - else {

258 - if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)

259 - buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1 %llu\n", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);

260 - else

261 - buffer_sprintf(wb, "netdata_info{instance=\"%s\",application=\"%s\",version=\"%s\"} 1\n", hostname, host->program_name, host->program_version);

262 -

263 - if(host->tags && *(host->tags)) {

264 - if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS) {

265 - buffer_sprintf(wb, "netdata_host_tags_info{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);

266 -

267 - // deprecated, exists only for compatibility with older queries

268 - buffer_sprintf(wb, "netdata_host_tags{%s} 1 %llu\n", host->tags, now_realtime_usec() / USEC_PER_MS);

269 - }

270 - else {

271 - buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", host->tags);

272 -

273 - // deprecated, exists only for compatibility with older queries

274 - buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", host->tags);

275 - }

276 - }

277 - }

278 -

279 - // send custom variables set for the host

280 - if(output_options & BACKENDS_PROMETHEUS_OUTPUT_VARIABLES){

281 - struct host_variables_callback_options opts = {

282 - .host = host,

283 - .wb = wb,

284 - .labels = (labels[0] == ',')?&labels[1]:labels,

285 - .backend_options = backend_options,

286 - .output_options = output_options,

287 - .prefix = prefix,

288 - .now = now_realtime_sec(),

289 - .host_header_printed = 0

290 - };

291 - foreach_host_variable_callback(host, print_host_variables, &opts);

292 - }

293 -

294 - // for each chart

295 - RRDSET *st;

296 - rrdset_foreach_read(st, host) {

297 - char chart[PROMETHEUS_ELEMENT_MAX + 1];

298 - char context[PROMETHEUS_ELEMENT_MAX + 1];

299 - char family[PROMETHEUS_ELEMENT_MAX + 1];

300 - char units[PROMETHEUS_ELEMENT_MAX + 1] = "";

301 -

302 - backends_prometheus_label_copy(chart, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);

303 - backends_prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);

304 - backends_prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);

305 -

306 - if(likely(backends_can_send_rrdset(backend_options, st))) {

307 - rrdset_rdlock(st);

308 -

309 - int as_collected = (BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED);

310 - int homogeneous = 1;

311 - if(as_collected) {

312 - if(rrdset_flag_check(st, RRDSET_FLAG_HOMOGENEOUS_CHECK))

313 - rrdset_update_heterogeneous_flag(st);

314 -

315 - if(rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS))

316 - homogeneous = 0;

317 - }

318 - else {

319 - if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE && !(output_options & BACKENDS_PROMETHEUS_OUTPUT_HIDEUNITS))

320 - backends_prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, output_options & BACKENDS_PROMETHEUS_OUTPUT_OLDUNITS);

321 - }

322 -

323 - if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))

324 - buffer_sprintf(wb, "\n# COMMENT %s chart \"%s\", context \"%s\", family \"%s\", units \"%s\"\n"

325 - , (homogeneous)?"homogeneous":"heterogeneous"

326 - , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id

327 - , st->context

328 - , st->family

329 - , st->units

330 - );

331 -

332 - // for each dimension

333 - RRDDIM *rd;

334 - rrddim_foreach_read(rd, st) {

335 - if(rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {

336 - char dimension[PROMETHEUS_ELEMENT_MAX + 1];

337 - char *suffix = "";

338 -

339 - if (as_collected) {

340 - // we need as-collected / raw data

341 -

342 - if(unlikely(rd->last_collected_time.tv_sec < after))

343 - continue;

344 -

345 - const char *t = "gauge", *h = "gives";

346 - if(rd->algorithm == RRD_ALGORITHM_INCREMENTAL ||

347 - rd->algorithm == RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL) {

348 - t = "counter";

349 - h = "delta gives";

350 - suffix = "_total";

351 - }

352 -

353 - if(homogeneous) {

354 - // all the dimensions of the chart, has the same algorithm, multiplier and divisor

355 - // we add all dimensions as labels

356 -

357 - backends_prometheus_label_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

358 -

359 - if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))

360 - buffer_sprintf(wb

361 - , "# COMMENT %s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"

362 - , prefix

363 - , context

364 - , suffix

365 - , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id

366 - , st->context

367 - , st->family

368 - , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id

369 - , rd->multiplier

370 - , rd->divisor

371 - , h

372 - , st->units

373 - , t

374 - );

375 -

376 - if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_TYPES))

377 - buffer_sprintf(wb, "# TYPE %s_%s%s %s\n"

378 - , prefix

379 - , context

380 - , suffix

381 - , t

382 - );

383 -

384 - if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)

385 - buffer_sprintf(wb

386 - , "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n"

387 - , prefix

388 - , context

389 - , suffix

390 - , chart

391 - , family

392 - , dimension

393 - , labels

394 - , rd->last_collected_value

395 - , timeval_msec(&rd->last_collected_time)

396 - );

397 - else

398 - buffer_sprintf(wb

399 - , "%s_%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " COLLECTED_NUMBER_FORMAT "\n"

400 - , prefix

401 - , context

402 - , suffix

403 - , chart

404 - , family

405 - , dimension

406 - , labels

407 - , rd->last_collected_value

408 - );

409 - }

410 - else {

411 - // the dimensions of the chart, do not have the same algorithm, multiplier or divisor

412 - // we create a metric per dimension

413 -

414 - backends_prometheus_name_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

415 -

416 - if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))

417 - buffer_sprintf(wb

418 - , "# COMMENT %s_%s_%s%s: chart \"%s\", context \"%s\", family \"%s\", dimension \"%s\", value * " COLLECTED_NUMBER_FORMAT " / " COLLECTED_NUMBER_FORMAT " %s %s (%s)\n"

419 - , prefix

420 - , context

421 - , dimension

422 - , suffix

423 - , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && st->name) ? st->name : st->id

424 - , st->context

425 - , st->family

426 - , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id

427 - , rd->multiplier

428 - , rd->divisor

429 - , h

430 - , st->units

431 - , t

432 - );

433 -

434 - if(unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_TYPES))

435 - buffer_sprintf(wb, "# TYPE %s_%s_%s%s %s\n"

436 - , prefix

437 - , context

438 - , dimension

439 - , suffix

440 - , t

441 - );

442 -

443 - if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)

444 - buffer_sprintf(wb

445 - , "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT " %llu\n"

446 - , prefix

447 - , context

448 - , dimension

449 - , suffix

450 - , chart

451 - , family

452 - , labels

453 - , rd->last_collected_value

454 - , timeval_msec(&rd->last_collected_time)

455 - );

456 - else

457 - buffer_sprintf(wb

458 - , "%s_%s_%s%s{chart=\"%s\",family=\"%s\"%s} " COLLECTED_NUMBER_FORMAT "\n"

459 - , prefix

460 - , context

461 - , dimension

462 - , suffix

463 - , chart

464 - , family

465 - , labels

466 - , rd->last_collected_value

467 - );

468 - }

469 - }

470 - else {

471 - // we need average or sum of the data

472 -

473 - time_t first_t = after, last_t = before;

474 - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);

475 -

476 - if(!isnan(value) && !isinf(value)) {

477 -

478 - if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE)

479 - suffix = "_average";

480 - else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM)

481 - suffix = "_sum";

482 -

483 - backends_prometheus_label_copy(dimension, (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

484 -

485 - if (unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP))

486 - buffer_sprintf(wb, "# COMMENT %s_%s%s%s: dimension \"%s\", value is %s, gauge, dt %llu to %llu inclusive\n"

487 - , prefix

488 - , context

489 - , units

490 - , suffix

491 - , (output_options & BACKENDS_PROMETHEUS_OUTPUT_NAMES && rd->name) ? rd->name : rd->id

492 - , st->units

493 - , (unsigned long long)first_t

494 - , (unsigned long long)last_t

495 - );

496 -

497 - if (unlikely(output_options & BACKENDS_PROMETHEUS_OUTPUT_TYPES))

498 - buffer_sprintf(wb, "# TYPE %s_%s%s%s gauge\n"

499 - , prefix

500 - , context

501 - , units

502 - , suffix

503 - );

504 -

505 - if(output_options & BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS)

506 - buffer_sprintf(wb, "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT " %llu\n"

507 - , prefix

508 - , context

509 - , units

510 - , suffix

511 - , chart

512 - , family

513 - , dimension

514 - , labels

515 - , value

516 - , last_t * MSEC_PER_SEC

517 - );

518 - else

519 - buffer_sprintf(wb, "%s_%s%s%s{chart=\"%s\",family=\"%s\",dimension=\"%s\"%s} " CALCULATED_NUMBER_FORMAT "\n"

520 - , prefix

521 - , context

522 - , units

523 - , suffix

524 - , chart

525 - , family

526 - , dimension

527 - , labels

528 - , value

529 - );

530 - }

531 - }

532 - }

533 - }

534 -

535 - rrdset_unlock(st);

536 - }

537 - }

538 -

539 - rrdhost_unlock(host);

540 -}

541 -

542 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

543 -inline static void remote_write_split_words(char *str, char **words, int max_words) {

544 - char *s = str;

545 - int i = 0;

546 -

547 - while(*s && i < max_words - 1) {

548 - while(*s && isspace(*s)) s++; // skip spaces to the beginning of a tag name

549 -

550 - if(*s)

551 - words[i] = s;

552 -

553 - while(*s && !isspace(*s) && *s != '=') s++; // find the end of the tag name

554 -

555 - if(*s != '=') {

556 - words[i] = NULL;

557 - break;

558 - }

559 - *s = '\0';

560 - s++;

561 - i++;

562 -

563 - while(*s && isspace(*s)) s++; // skip spaces to the beginning of a tag value

564 -

565 - if(*s && *s == '"') s++; // strip an opening quote

566 - if(*s)

567 - words[i] = s;

568 -

569 - while(*s && !isspace(*s) && *s != ',') s++; // find the end of the tag value

570 -

571 - if(*s && *s != ',') {

572 - words[i] = NULL;

573 - break;

574 - }

575 - if(s != words[i] && *(s - 1) == '"') *(s - 1) = '\0'; // strip a closing quote

576 - if(*s != '\0') {

577 - *s = '\0';

578 - s++;

579 - i++;

580 - }

581 - }

582 -}

583 -

584 -void backends_rrd_stats_remote_write_allmetrics_prometheus(

585 - RRDHOST *host

586 - , const char *__hostname

587 - , const char *prefix

588 - , BACKEND_OPTIONS backend_options

589 - , time_t after

590 - , time_t before

591 - , size_t *count_charts

592 - , size_t *count_dims

593 - , size_t *count_dims_skipped

594 -) {

595 - char hostname[PROMETHEUS_ELEMENT_MAX + 1];

596 - backends_prometheus_label_copy(hostname, __hostname, PROMETHEUS_ELEMENT_MAX);

597 -

598 - backends_add_host_info("netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);

599 -

600 - if(host->tags && *(host->tags)) {

601 - char tags[PROMETHEUS_LABELS_MAX + 1];

602 - strncpy(tags, host->tags, PROMETHEUS_LABELS_MAX);

603 - char *words[PROMETHEUS_LABELS_MAX_NUMBER] = {NULL};

604 - int i;

605 -

606 - remote_write_split_words(tags, words, PROMETHEUS_LABELS_MAX_NUMBER);

607 -

608 - backends_add_host_info("netdata_host_tags_info", hostname, NULL, NULL, now_realtime_usec() / USEC_PER_MS);

609 -

610 - for(i = 0; words[i] != NULL && words[i + 1] != NULL && (i + 1) < PROMETHEUS_LABELS_MAX_NUMBER; i += 2) {

611 - backends_add_tag(words[i], words[i + 1]);

612 - }

613 - }

614 -

615 - // for each chart

616 - RRDSET *st;

617 - rrdset_foreach_read(st, host) {

618 - char chart[PROMETHEUS_ELEMENT_MAX + 1];

619 - char context[PROMETHEUS_ELEMENT_MAX + 1];

620 - char family[PROMETHEUS_ELEMENT_MAX + 1];

621 - char units[PROMETHEUS_ELEMENT_MAX + 1] = "";

622 -

623 - backends_prometheus_label_copy(chart, (backend_options & BACKEND_OPTION_SEND_NAMES && st->name)?st->name:st->id, PROMETHEUS_ELEMENT_MAX);

624 - backends_prometheus_label_copy(family, st->family, PROMETHEUS_ELEMENT_MAX);

625 - backends_prometheus_name_copy(context, st->context, PROMETHEUS_ELEMENT_MAX);

626 -

627 - if(likely(backends_can_send_rrdset(backend_options, st))) {

628 - rrdset_rdlock(st);

629 -

630 - (*count_charts)++;

631 -

632 - int as_collected = (BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED);

633 - int homogeneous = 1;

634 - if(as_collected) {

635 - if(rrdset_flag_check(st, RRDSET_FLAG_HOMOGENEOUS_CHECK))

636 - rrdset_update_heterogeneous_flag(st);

637 -

638 - if(rrdset_flag_check(st, RRDSET_FLAG_HETEROGENEOUS))

639 - homogeneous = 0;

640 - }

641 - else {

642 - if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE)

643 - backends_prometheus_units_copy(units, st->units, PROMETHEUS_ELEMENT_MAX, 0);

644 - }

645 -

646 - // for each dimension

647 - RRDDIM *rd;

648 - rrddim_foreach_read(rd, st) {

649 - if(rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {

650 - char name[PROMETHEUS_LABELS_MAX + 1];

651 - char dimension[PROMETHEUS_ELEMENT_MAX + 1];

652 - char *suffix = "";

653 -

654 - if (as_collected) {

655 - // we need as-collected / raw data

656 -

657 - if(unlikely(rd->last_collected_time.tv_sec < after)) {

658 - debug(D_BACKEND, "BACKEND: not sending dimension '%s' of chart '%s' from host '%s', its last data collection (%lu) is not within our timeframe (%lu to %lu)", rd->id, st->id, __hostname, (unsigned long)rd->last_collected_time.tv_sec, (unsigned long)after, (unsigned long)before);

659 - (*count_dims_skipped)++;

660 - continue;

661 - }

662 -

663 - if(homogeneous) {

664 - // all the dimensions of the chart, has the same algorithm, multiplier and divisor

665 - // we add all dimensions as labels

666 -

667 - backends_prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

668 - snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s", prefix, context, suffix);

669 -

670 - backends_add_metric(name, chart, family, dimension, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time));

671 - (*count_dims)++;

672 - }

673 - else {

674 - // the dimensions of the chart, do not have the same algorithm, multiplier or divisor

675 - // we create a metric per dimension

676 -

677 - backends_prometheus_name_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

678 - snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s_%s%s", prefix, context, dimension, suffix);

679 -

680 - backends_add_metric(name, chart, family, NULL, hostname, rd->last_collected_value, timeval_msec(&rd->last_collected_time));

681 - (*count_dims)++;

682 - }

683 - }

684 - else {

685 - // we need average or sum of the data

686 -

687 - time_t first_t = after, last_t = before;

688 - calculated_number value = backend_calculate_value_from_stored_data(st, rd, after, before, backend_options, &first_t, &last_t);

689 -

690 - if(!isnan(value) && !isinf(value)) {

691 -

692 - if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE)

693 - suffix = "_average";

694 - else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM)

695 - suffix = "_sum";

696 -

697 - backends_prometheus_label_copy(dimension, (backend_options & BACKEND_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id, PROMETHEUS_ELEMENT_MAX);

698 - snprintf(name, PROMETHEUS_LABELS_MAX, "%s_%s%s%s", prefix, context, units, suffix);

699 -

700 - backends_add_metric(name, chart, family, dimension, hostname, value, last_t * MSEC_PER_SEC);

701 - (*count_dims)++;

702 - }

703 - }

704 - }

705 - }

706 -

707 - rrdset_unlock(st);

708 - }

709 - }

710 -}

711 -#endif /* ENABLE_PROMETHEUS_REMOTE_WRITE */

712 -

713 -static inline time_t prometheus_preparation(RRDHOST *host, BUFFER *wb, BACKEND_OPTIONS backend_options, const char *server, time_t now, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options) {

714 - if(!server || !*server) server = "default";

715 -

716 - time_t after = prometheus_server_last_access(server, host, now);

717 -

718 - int first_seen = 0;

719 - if(!after) {

720 - after = now - global_backend_update_every;

721 - first_seen = 1;

722 - }

723 -

724 - if(after > now) {

725 - // oops! this should never happen

726 - after = now - global_backend_update_every;

727 - }

728 -

729 - if(output_options & BACKENDS_PROMETHEUS_OUTPUT_HELP) {

730 - char *mode;

731 - if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AS_COLLECTED)

732 - mode = "as collected";

733 - else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_AVERAGE)

734 - mode = "average";

735 - else if(BACKEND_OPTIONS_DATA_SOURCE(backend_options) == BACKEND_SOURCE_DATA_SUM)

736 - mode = "sum";

737 - else

738 - mode = "unknown";

739 -

740 - buffer_sprintf(wb, "# COMMENT netdata \"%s\" to %sprometheus \"%s\", source \"%s\", last seen %lu %s, time range %lu to %lu\n\n"

741 - , host->hostname

742 - , (first_seen)?"FIRST SEEN ":""

743 - , server

744 - , mode

745 - , (unsigned long)((first_seen)?0:(now - after))

746 - , (first_seen)?"never":"seconds ago"

747 - , (unsigned long)after, (unsigned long)now

748 - );

749 - }

750 -

751 - return after;

752 -}

753 -

754 -void backends_rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options) {

755 - time_t before = now_realtime_sec();

756 -

757 - // we start at the point we had stopped before

758 - time_t after = prometheus_preparation(host, wb, backend_options, server, before, output_options);

759 -

760 - rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, backend_options, after, before, 0, output_options);

761 -}

762 -

763 -void backends_rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options) {

764 - time_t before = now_realtime_sec();

765 -

766 - // we start at the point we had stopped before

767 - time_t after = prometheus_preparation(host, wb, backend_options, server, before, output_options);

768 -

769 - rrd_rdlock();

770 - rrdhost_foreach_read(host) {

771 - rrd_stats_api_v1_charts_allmetrics_prometheus(host, wb, prefix, backend_options, after, before, 1, output_options);

772 - }

773 - rrd_unlock();

774 -}

775 -

776 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

777 -int backends_process_prometheus_remote_write_response(BUFFER *b) {

778 - if(unlikely(!b)) return 1;

779 -

780 - const char *s = buffer_tostring(b);

781 - int len = buffer_strlen(b);

782 -

783 - // do nothing with HTTP responses 200 or 204

784 -

785 - while(!isspace(*s) && len) {

786 - s++;

787 - len--;

788 - }

789 - s++;

790 - len--;

791 -

792 - if(likely(len > 4 && (!strncmp(s, "200 ", 4) || !strncmp(s, "204 ", 4))))

793 - return 0;

794 - else

795 - return discard_response(b, "prometheus remote write");

796 -}

797 -#endif

backends/prometheus/backend_prometheus.h deleted

-37

@@ -1,37 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_BACKEND_PROMETHEUS_H

4 -#define NETDATA_BACKEND_PROMETHEUS_H 1

5 -

6 -#include "backends/backends.h"

7 -

8 -typedef enum backends_prometheus_output_flags {

9 - BACKENDS_PROMETHEUS_OUTPUT_NONE = 0,

10 - BACKENDS_PROMETHEUS_OUTPUT_HELP = (1 << 0),

11 - BACKENDS_PROMETHEUS_OUTPUT_TYPES = (1 << 1),

12 - BACKENDS_PROMETHEUS_OUTPUT_NAMES = (1 << 2),

13 - BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS = (1 << 3),

14 - BACKENDS_PROMETHEUS_OUTPUT_VARIABLES = (1 << 4),

15 - BACKENDS_PROMETHEUS_OUTPUT_OLDUNITS = (1 << 5),

16 - BACKENDS_PROMETHEUS_OUTPUT_HIDEUNITS = (1 << 6)

17 -} BACKENDS_PROMETHEUS_OUTPUT_OPTIONS;

18 -

19 -extern void backends_rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options);

20 -extern void backends_rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(RRDHOST *host, BUFFER *wb, const char *server, const char *prefix, BACKEND_OPTIONS backend_options, BACKENDS_PROMETHEUS_OUTPUT_OPTIONS output_options);

21 -

22 -#if ENABLE_PROMETHEUS_REMOTE_WRITE

23 -extern void backends_rrd_stats_remote_write_allmetrics_prometheus(

24 - RRDHOST *host

25 - , const char *__hostname

26 - , const char *prefix

27 - , BACKEND_OPTIONS backend_options

28 - , time_t after

29 - , time_t before

30 - , size_t *count_charts

31 - , size_t *count_dims

32 - , size_t *count_dims_skipped

33 -);

34 -extern int backends_process_prometheus_remote_write_response(BUFFER *b);

35 -#endif

36 -

37 -#endif //NETDATA_BACKEND_PROMETHEUS_H

backends/prometheus/remote_write/Makefile.am deleted

-14

@@ -1,14 +0,0 @@

1 -# SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -AUTOMAKE_OPTIONS = subdir-objects

4 -MAINTAINERCLEANFILES = $(srcdir)/Makefile.in

5 -

6 -CLEANFILES = \

7 - remote_write.pb.cc \

8 - remote_write.pb.h \

9 - $(NULL)

10 -

11 -dist_noinst_DATA = \

12 - remote_write.proto \

13 - README.md \

14 - $(NULL)

backends/prometheus/remote_write/README.md deleted

-41

@@ -1,41 +0,0 @@

1 -<!--

2 -title: "Prometheus remote write backend"

3 -custom_edit_url: https://github.com/netdata/netdata/edit/master/backends/prometheus/remote_write/README.md

4 --->

5 -

6 -# Prometheus remote write backend

7 -

8 -## Prerequisites

9 -

10 -To use the prometheus remote write API with [storage

11 -providers](https://prometheus.io/docs/operating/integrations/#remote-endpoints-and-storage)

12 -[protobuf](https://developers.google.com/protocol-buffers/) and [snappy](https://github.com/google/snappy) libraries

13 -should be installed first. Next, Netdata should be re-installed from the source. The installer will detect that the

14 -required libraries and utilities are now available.

15 -

16 -## Configuration

17 -

18 -An additional option in the backend configuration section is available for the remote write backend:

19 -

20 -```conf

21 -[backend]

22 - remote write URL path = /receive

23 -```

24 -

25 -The default value is `/receive`. `remote write URL path` is used to set an endpoint path for the remote write protocol.

26 -For example, if your endpoint is `http://example.domain:example_port/storage/read` you should set

27 -

28 -```conf

29 -[backend]

30 - destination = example.domain:example_port

31 - remote write URL path = /storage/read

32 -```

33 -

34 -`buffered` and `lost` dimensions in the Netdata Backend Data Size operation monitoring chart estimate uncompressed

35 -buffer size on failures.

36 -

37 -## Notes

38 -

39 -The remote write backend does not support `buffer on failures`

40 -

41 -

backends/prometheus/remote_write/remote_write.cc deleted

-120

@@ -1,120 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#include <snappy.h>

4 -#include "exporting/prometheus/remote_write/remote_write.pb.h"

5 -#include "remote_write.h"

6 -

7 -using namespace prometheus;

8 -

9 -static google::protobuf::Arena arena;

10 -static WriteRequest *write_request;

11 -

12 -void backends_init_write_request() {

13 - GOOGLE_PROTOBUF_VERIFY_VERSION;

14 - write_request = google::protobuf::Arena::CreateMessage<WriteRequest>(&arena);

15 -}

16 -

17 -void backends_clear_write_request() {

18 - write_request->clear_timeseries();

19 -}

20 -

21 -void backends_add_host_info(const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp) {

22 - TimeSeries *timeseries;

23 - Sample *sample;

24 - Label *label;

25 -

26 - timeseries = write_request->add_timeseries();

27 -

28 - label = timeseries->add_labels();

29 - label->set_name("__name__");

30 - label->set_value(name);

31 -

32 - label = timeseries->add_labels();

33 - label->set_name("instance");

34 - label->set_value(instance);

35 -

36 - if(application) {

37 - label = timeseries->add_labels();

38 - label->set_name("application");

39 - label->set_value(application);

40 - }

41 -

42 - if(version) {

43 - label = timeseries->add_labels();

44 - label->set_name("version");

45 - label->set_value(version);

46 - }

47 -

48 - sample = timeseries->add_samples();

49 - sample->set_value(1);

50 - sample->set_timestamp(timestamp);

51 -}

52 -

53 -// adds tag to the last created timeseries

54 -void backends_add_tag(char *tag, char *value) {

55 - TimeSeries *timeseries;

56 - Label *label;

57 -

58 - timeseries = write_request->mutable_timeseries(write_request->timeseries_size() - 1);

59 -

60 - label = timeseries->add_labels();

61 - label->set_name(tag);

62 - label->set_value(value);

63 -}

64 -

65 -void backends_add_metric(const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp) {

66 - TimeSeries *timeseries;

67 - Sample *sample;

68 - Label *label;

69 -

70 - timeseries = write_request->add_timeseries();

71 -

72 - label = timeseries->add_labels();

73 - label->set_name("__name__");

74 - label->set_value(name);

75 -

76 - label = timeseries->add_labels();

77 - label->set_name("chart");

78 - label->set_value(chart);

79 -

80 - label = timeseries->add_labels();

81 - label->set_name("family");

82 - label->set_value(family);

83 -

84 - if(dimension) {

85 - label = timeseries->add_labels();

86 - label->set_name("dimension");

87 - label->set_value(dimension);

88 - }

89 -

90 - label = timeseries->add_labels();

91 - label->set_name("instance");

92 - label->set_value(instance);

93 -

94 - sample = timeseries->add_samples();

95 - sample->set_value(value);

96 - sample->set_timestamp(timestamp);

97 -}

98 -

99 -size_t backends_get_write_request_size(){

100 -#if GOOGLE_PROTOBUF_VERSION < 3001000

101 - size_t size = (size_t)snappy::MaxCompressedLength(write_request->ByteSize());

102 -#else

103 - size_t size = (size_t)snappy::MaxCompressedLength(write_request->ByteSizeLong());

104 -#endif

105 -

106 - return (size < INT_MAX)?size:0;

107 -}

108 -

109 -int backends_pack_write_request(char *buffer, size_t *size) {

110 - std::string uncompressed_write_request;

111 - if(write_request->SerializeToString(&uncompressed_write_request) == false) return 1;

112 -

113 - snappy::RawCompress(uncompressed_write_request.data(), uncompressed_write_request.size(), buffer, size);

114 -

115 - return 0;

116 -}

117 -

118 -void backends_protocol_buffers_shutdown() {

119 - google::protobuf::ShutdownProtobufLibrary();

120 -}

backends/prometheus/remote_write/remote_write.h deleted

-30

@@ -1,30 +0,0 @@

1 -// SPDX-License-Identifier: GPL-3.0-or-later

2 -

3 -#ifndef NETDATA_BACKEND_PROMETHEUS_REMOTE_WRITE_H

4 -#define NETDATA_BACKEND_PROMETHEUS_REMOTE_WRITE_H

5 -

6 -#ifdef __cplusplus

7 -extern "C" {

8 -#endif

9 -

10 -void backends_init_write_request();

11 -

12 -void backends_clear_write_request();

13 -

14 -void backends_add_host_info(const char *name, const char *instance, const char *application, const char *version, const int64_t timestamp);

15 -

16 -void backends_add_tag(char *tag, char *value);

17 -

18 -void backends_add_metric(const char *name, const char *chart, const char *family, const char *dimension, const char *instance, const double value, const int64_t timestamp);

19 -

20 -size_t backends_get_write_request_size();

21 -

22 -int backends_pack_write_request(char *buffer, size_t *size);

23 -

24 -void backends_protocol_buffers_shutdown();

25 -

26 -#ifdef __cplusplus

27 -}

28 -#endif

29 -

30 -#endif //NETDATA_BACKEND_PROMETHEUS_REMOTE_WRITE_H

backends/prometheus/remote_write/remote_write.proto deleted

-29

@@ -1,29 +0,0 @@

1 -syntax = "proto3";

2 -package prometheus;

3 -

4 -option cc_enable_arenas = true;

5 -

6 -import "google/protobuf/descriptor.proto";

7 -

8 -message WriteRequest {

9 - repeated TimeSeries timeseries = 1 [(nullable) = false];

10 -}

11 -

12 -message TimeSeries {

13 - repeated Label labels = 1 [(nullable) = false];

14 - repeated Sample samples = 2 [(nullable) = false];

15 -}

16 -

17 -message Label {

18 - string name = 1;

19 - string value = 2;

20 -}

21 -

22 -message Sample {

23 - double value = 1;

24 - int64 timestamp = 2;

25 -}

26 -

27 -extend google.protobuf.FieldOptions {

28 - bool nullable = 65001;

29 -}

build_external/scenarios/aclk-testing/agent_netdata.conf

-15

@@ -138,21 +138,6 @@

138 [cloud]

139 # cloud base url = https://netdata.cloud

140

141 -[backend]

142 - # host tags =

143 - # enabled = no

144 - # data source = average

145 - # type = graphite

146 - # destination = localhost

147 - # prefix = netdata

148 - # hostname = b073e16793c4

149 - # update every = 10

150 - # buffer on failures = 10

151 - # timeout ms = 20000

152 - # send names instead of ids = yes

153 - # send charts matching = *

154 - # send hosts matching = localhost *

155 -

141 [statsd]

142 # enabled = yes

143 # update every (flushInterval) = 1

collectors/README.md

+3 -5

@@ -40,10 +40,8 @@ our [collectors' configuration reference](/collectors/REFERENCE.md).

40 **[Dashboards](/web/README.md)**: Visualize your newly-collect metrics in real-time using Netdata's [built-in

41 dashboard](/web/gui/README.md).

43 -**[Backends](/backends/README.md)**: Extend our built-in [database engine](/database/engine/README.md), which supports

44 -long-term metrics storage, by archiving metrics to like Graphite, Prometheus, MongoDB, TimescaleDB, and more.

45 -

46 -**[Exporting](/exporting/README.md)**: An experimental refactoring of our backends system with a modular system and

47 -support for exporting metrics to multiple systems simultaneously.

43 +**[Exporting](/exporting/README.md)**: Extend our built-in [database engine](/database/engine/README.md), which supports

44 +long-term metrics storage, by archiving metrics to external databases like Graphite, Prometheus, MongoDB, TimescaleDB, and more.

45 +It can export metrics to multiple databases simultaneously.

collectors/plugins.d/README.md

+2 -2

@@ -231,7 +231,7 @@ the template is:

231

232 - `options`

233

234 - a space separated list of options, enclosed in quotes. 4 options are currently supported: `obsolete` to mark a chart as obsolete (Netdata will hide it and delete it after some time), `detail` to mark a chart as insignificant (this may be used by dashboards to make the charts smaller, or somehow visualize properly a less important chart), `store_first` to make Netdata store the first collected value, assuming there was an invisible previous value set to zero (this is used by statsd charts - if the first data collected value of incremental dimensions is not zero based, unrealistic spikes will appear with this option set) and `hidden` to perform all operations on a chart, but do not offer it on dashboards (the chart will be send to backends). `CHART` options have been added in Netdata v1.7 and the `hidden` option was added in 1.10.

234 + a space separated list of options, enclosed in quotes. 4 options are currently supported: `obsolete` to mark a chart as obsolete (Netdata will hide it and delete it after some time), `detail` to mark a chart as insignificant (this may be used by dashboards to make the charts smaller, or somehow visualize properly a less important chart), `store_first` to make Netdata store the first collected value, assuming there was an invisible previous value set to zero (this is used by statsd charts - if the first data collected value of incremental dimensions is not zero based, unrealistic spikes will appear with this option set) and `hidden` to perform all operations on a chart, but do not offer it on dashboards (the chart will be send to external databases). `CHART` options have been added in Netdata v1.7 and the `hidden` option was added in 1.10.

235

236 - `plugin` and `module`

237

@@ -252,7 +252,7 @@ the template is:

252 the `id` of this dimension (it is a text value, not numeric),

253 this will be needed later to add values to the dimension

254

255 - We suggest to avoid using `.` in dimension ids. Backends expect metrics to be `.` separated and people will get confused if a dimension id contains a dot.

255 + We suggest to avoid using `.` in dimension ids. External databases expect metrics to be `.` separated and people will get confused if a dimension id contains a dot.

256

257 - `name`

258

collectors/python.d.plugin/beanstalk/beanstalk.conf

+1 -1

@@ -43,7 +43,7 @@

43 # When a plugin sends the obsolete flag, the charts are not deleted

44 # from netdata immediately.

45 # They will be hidden immediately (not offered to dashboard viewer,

46 -# streamed upstream and archived to backends) and deleted one hour

46 +# streamed upstream and archived to external databases) and deleted one hour

47 # later (configurable from netdata.conf).

48 # chart_cleanup: 10

collectors/python.d.plugin/httpcheck/httpcheck.conf

+1 -1

@@ -35,7 +35,7 @@

35 # A chart is marked as obsolete if it has not been updated

36 # 'chart_cleanup' iterations in a row.

37 # They will be hidden immediately (not offered to dashboard viewer,

38 -# streamed upstream and archived to backends) and deleted one hour

38 +# streamed upstream and archived to external databases) and deleted one hour

39 # later (configurable from netdata.conf).

40 # -- For this plugin, cleanup MUST be disabled, otherwise we lose response

41 # time charts

collectors/python.d.plugin/portcheck/portcheck.conf

+1 -1

@@ -35,7 +35,7 @@

35 # A chart is marked as obsolete if it has not been updated

36 # 'chart_cleanup' iterations in a row.

37 # They will be hidden immediately (not offered to dashboard viewer,

38 -# streamed upstream and archived to backends) and deleted one hour

38 +# streamed upstream and archived to external databases) and deleted one hour

39 # later (configurable from netdata.conf).

40 # -- For this plugin, cleanup MUST be disabled, otherwise we lose latency chart

41 chart_cleanup: 0

configure.ac

+43 -51

@@ -71,10 +71,10 @@ AC_ARG_ENABLE(

71 [enable_plugin_xenstat="detect"]

72 )

73 AC_ARG_ENABLE(

74 - [backend-kinesis],

75 - [AS_HELP_STRING([--enable-backend-kinesis], [enable kinesis backend @<:@default autodetect@:>@])],

74 + [exporting-kinesis],

75 + [AS_HELP_STRING([--enable-exporting-kinesis], [enable kinesis exporting connector @<:@default autodetect@:>@])],

76 ,

77 - [enable_backend_kinesis="detect"]

77 + [enable_exporting_kinesis="detect"]

78 )

79 AC_ARG_ENABLE(

80 [exporting-pubsub],

@@ -83,16 +83,16 @@ AC_ARG_ENABLE(

83 [enable_exporting_pubsub="detect"]

84 )

85 AC_ARG_ENABLE(

86 - [backend-prometheus-remote-write],

87 - [AS_HELP_STRING([--enable-backend-prometheus-remote-write], [enable prometheus remote write backend @<:@default autodetect@:>@])],

86 + [exporting-prometheus-remote-write],

87 + [AS_HELP_STRING([--enable-exporting-prometheus-remote-write], [enable prometheus remote write exporting connector @<:@default autodetect@:>@])],

88 ,

89 - [enable_backend_prometheus_remote_write="detect"]

89 + [enable_exporting_prometheus_remote_write="detect"]

90 )

91 AC_ARG_ENABLE(

92 - [backend-mongodb],

93 - [AS_HELP_STRING([--enable-backend-mongodb], [enable mongodb backend @<:@default autodetect@:>@])],

92 + [exporting-mongodb],

93 + [AS_HELP_STRING([--enable-exporting-mongodb], [enable mongodb exporting @<:@default autodetect@:>@])],

94 ,

95 - [enable_backend_mongodb="detect"]

95 + [enable_exporting_mongodb="detect"]

96 )

97 AC_ARG_ENABLE(

98 [pedantic],

@@ -1246,7 +1246,7 @@ AM_CONDITIONAL([ENABLE_PLUGIN_SLABINFO], [test "${enable_plugin_slabinfo}" = "ye

1246

1247

1248 # -----------------------------------------------------------------------------

1249 -# AWS Kinesis backend - libaws-cpp-sdk-kinesis, libaws-cpp-sdk-core, libssl, libcrypto, libcurl

1249 +# AWS Kinesis exporting connector - libaws-cpp-sdk-kinesis, libaws-cpp-sdk-core, libssl, libcrypto, libcurl

1250

1251 PKG_CHECK_MODULES(

1252 [LIBCRYPTO],

@@ -1298,39 +1298,39 @@ PKG_CHECK_MODULES(

1298 [have_libaws_cpp_sdk_kinesis=no]

1299 )

1300

1301 -test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_kinesis}" != "yes" && \

1301 +test "${enable_exporting_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_kinesis}" != "yes" && \

1302 AC_MSG_ERROR([libaws-cpp-sdk-kinesis required but not found. try installing AWS C++ SDK])

1303

1304 -test "${enable_backend_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" != "yes" && \

1304 +test "${enable_exporting_kinesis}" = "yes" -a "${have_libaws_cpp_sdk_core}" != "yes" && \

1305 AC_MSG_ERROR([libaws-cpp-sdk-core required but not found. try installing AWS C++ SDK])

1306

1307 -test "${enable_backend_kinesis}" = "yes" -a "${have_libcurl}" != "yes" && \

1307 +test "${enable_exporting_kinesis}" = "yes" -a "${have_libcurl}" != "yes" && \

1308 AC_MSG_ERROR([libcurl required but not found])

1309

1310 -test "${enable_backend_kinesis}" = "yes" -a "${have_libssl}" != "yes" && \

1310 +test "${enable_exporting_kinesis}" = "yes" -a "${have_libssl}" != "yes" && \

1311 AC_MSG_ERROR([libssl required but not found])

1312

1313 -test "${enable_backend_kinesis}" = "yes" -a "${have_libcrypto}" != "yes" && \

1313 +test "${enable_exporting_kinesis}" = "yes" -a "${have_libcrypto}" != "yes" && \

1314 AC_MSG_ERROR([libcrypto required but not found])

1315

1316 -AC_MSG_CHECKING([if kinesis backend should be enabled])

1317 -if test "${enable_backend_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" \

1316 +AC_MSG_CHECKING([if kinesis exporting connector should be enabled])

1317 +if test "${enable_exporting_kinesis}" != "no" -a "${have_libaws_cpp_sdk_kinesis}" = "yes" \

1318 -a "${have_libaws_cpp_sdk_core}" = "yes" \

1319 -a "${have_libcurl}" = "yes" \

1320 -a "${have_libssl}" = "yes" \

1321 -a "${have_libcrypto}" = "yes"; then

1322 - enable_backend_kinesis="yes"

1322 + enable_exporting_kinesis="yes"

1323 AC_DEFINE([HAVE_KINESIS], [1], [libaws-cpp-sdk-kinesis usability])

1324 OPTIONAL_KINESIS_CFLAGS="${LIBCRYPTO_CFLAGS} ${LIBSSL_CFLAGS} ${LIBCURL_CFLAGS}"

1325 CXX11FLAG="${AWS_CPP_SDK_KINESIS_CFLAGS} ${AWS_CPP_SDK_CORE_CFLAGS}"

1326 OPTIONAL_KINESIS_LIBS="${AWS_CPP_SDK_KINESIS_LIBS} ${AWS_CPP_SDK_CORE_LIBS} \

1327 ${LIBCRYPTO_LIBS} ${LIBSSL_LIBS} ${LIBCURL_LIBS}"

1328 else

1329 - enable_backend_kinesis="no"

1329 + enable_exporting_kinesis="no"

1330 fi

1331

1332 -AC_MSG_RESULT([${enable_backend_kinesis}])

1333 -AM_CONDITIONAL([ENABLE_BACKEND_KINESIS], [test "${enable_backend_kinesis}" = "yes"])

1332 +AC_MSG_RESULT([${enable_exporting_kinesis}])

1333 +AM_CONDITIONAL([ENABLE_EXPORTING_KINESIS], [test "${enable_exporting_kinesis}" = "yes"])

1334

1335

1336 # -----------------------------------------------------------------------------

@@ -1365,7 +1365,7 @@ test "${enable_pubsub}" = "yes" -a "${have_grpc}" != "yes" && \

1365 test "${enable_pubsub}" = "yes" -a "${have_pubsub_protos}" != "yes" && \

1366 AC_MSG_ERROR([libgoogleapis_cpp_pubsub_protos required but not found. try installing googleapis])

1367

1368 -test "${enable_backend_prometheus_remote_write}" = "yes" -a "${have_CXX_compiler}" != "yes" && \

1368 +test "${enable_exporting_prometheus_remote_write}" = "yes" -a "${have_CXX_compiler}" != "yes" && \

1369 AC_MSG_ERROR([C++ compiler required but not found. try installing g++])

1370

1371 AC_MSG_CHECKING([if pubsub exporting connector should be enabled])

@@ -1384,7 +1384,7 @@ AM_CONDITIONAL([ENABLE_EXPORTING_PUBSUB], [test "${enable_exporting_pubsub}" = "

1384

1385

1386 # -----------------------------------------------------------------------------

1387 -# Prometheus remote write backend - libprotobuf, libsnappy, protoc

1387 +# Prometheus remote write exporting connector - libprotobuf, libsnappy, protoc

1388

1389 AC_MSG_CHECKING([for snappy::RawCompress in -lsnappy])

1390

@@ -1421,22 +1421,22 @@ AC_MSG_CHECKING([for snappy::RawCompress in -lsnappy])

1421

1422 AC_MSG_RESULT([${have_libsnappy}])

1423

1424 -test "${enable_backend_prometheus_remote_write}" = "yes" -a "${have_libprotobuf}" != "yes" && \

1424 +test "${enable_exporting_prometheus_remote_write}" = "yes" -a "${have_libprotobuf}" != "yes" && \

1425 AC_MSG_ERROR([libprotobuf required but not found. try installing protobuf])

1426

1427 -test "${enable_backend_prometheus_remote_write}" = "yes" -a "${have_libsnappy}" != "yes" && \

1427 +test "${enable_exporting_prometheus_remote_write}" = "yes" -a "${have_libsnappy}" != "yes" && \

1428 AC_MSG_ERROR([libsnappy required but not found. try installing snappy])

1429

1430 -test "${enable_backend_prometheus_remote_write}" = "yes" -a "${have_protoc}" != "yes" && \

1430 +test "${enable_exporting_prometheus_remote_write}" = "yes" -a "${have_protoc}" != "yes" && \

1431 AC_MSG_ERROR([protoc compiler required but not found. try installing protobuf])

1432

1433 -test "${enable_backend_prometheus_remote_write}" = "yes" -a "${have_CXX_compiler}" != "yes" && \

1433 +test "${enable_exporting_prometheus_remote_write}" = "yes" -a "${have_CXX_compiler}" != "yes" && \

1434 AC_MSG_ERROR([C++ compiler required but not found. try installing g++])

1435

1436 -AC_MSG_CHECKING([if prometheus remote write backend should be enabled])

1437 -if test "${enable_backend_prometheus_remote_write}" != "no" -a "${have_libprotobuf}" = "yes" -a "${have_libsnappy}" = "yes" \

1436 +AC_MSG_CHECKING([if prometheus remote write exporting connector should be enabled])

1437 +if test "${enable_exporting_prometheus_remote_write}" != "no" -a "${have_libprotobuf}" = "yes" -a "${have_libsnappy}" = "yes" \

1438 -a "${have_protoc}" = "yes" -a "${have_CXX_compiler}" = "yes"; then

1439 - enable_backend_prometheus_remote_write="yes"

1439 + enable_exporting_prometheus_remote_write="yes"

1440 AC_DEFINE([ENABLE_PROMETHEUS_REMOTE_WRITE], [1], [Prometheus remote write API usability])

1441 OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS="${SNAPPY_CFLAGS} -I \$(abs_top_srcdir)/exporting/prometheus/remote_write"

1442 CXX11FLAG="-std=c++11"

@@ -1444,15 +1444,15 @@ if test "${enable_backend_prometheus_remote_write}" != "no" -a "${have_libprotob

1444 OPTIONAL_PROTOBUF_CFLAGS="${PROTOBUF_CFLAGS}"

1445 OPTIONAL_PROTOBUF_LIBS="${PROTOBUF_LIBS}"

1446 else

1447 - enable_backend_prometheus_remote_write="no"

1447 + enable_exporting_prometheus_remote_write="no"

1448 fi

1449

1450 -AC_MSG_RESULT([${enable_backend_prometheus_remote_write}])

1451 -AM_CONDITIONAL([ENABLE_BACKEND_PROMETHEUS_REMOTE_WRITE], [test "${enable_backend_prometheus_remote_write}" = "yes"])

1450 +AC_MSG_RESULT([${enable_exporting_prometheus_remote_write}])

1451 +AM_CONDITIONAL([ENABLE_EXPORTING_PROMETHEUS_REMOTE_WRITE], [test "${enable_exporting_prometheus_remote_write}" = "yes"])

1452

1453

1454 # -----------------------------------------------------------------------------

1455 -# MongoDB backend - libmongoc

1455 +# MongoDB exporting connector - libmongoc

1456

1457 PKG_CHECK_MODULES(

1458 [LIBMONGOC],

@@ -1461,21 +1461,21 @@ PKG_CHECK_MODULES(

1461 [have_libmongoc=no]

1462 )

1463

1464 -test "${enable_backend_mongodb}" = "yes" -a "${have_libmongoc}" != "yes" && \

1464 +test "${enable_exporting_mongodb}" = "yes" -a "${have_libmongoc}" != "yes" && \

1465 AC_MSG_ERROR([libmongoc required but not found. Try installing `mongoc`.])

1466

1467 -AC_MSG_CHECKING([if mongodb backend should be enabled])

1468 -if test "${enable_backend_mongodb}" != "no" -a "${have_libmongoc}" = "yes"; then

1469 - enable_backend_mongodb="yes"

1467 +AC_MSG_CHECKING([if mongodb exporting connector should be enabled])

1468 +if test "${enable_exporting_mongodb}" != "no" -a "${have_libmongoc}" = "yes"; then

1469 + enable_exporting_mongodb="yes"

1470 AC_DEFINE([HAVE_MONGOC], [1], [libmongoc usability])

1471 OPTIONAL_MONGOC_CFLAGS="${LIBMONGOC_CFLAGS}"

1472 OPTIONAL_MONGOC_LIBS="${LIBMONGOC_LIBS}"

1473 else

1474 - enable_backend_mongodb="no"

1474 + enable_exporting_mongodb="no"

1475 fi

1476

1477 -AC_MSG_RESULT([${enable_backend_mongodb}])

1478 -AM_CONDITIONAL([ENABLE_BACKEND_MONGODB], [test "${enable_backend_mongodb}" = "yes"])

1477 +AC_MSG_RESULT([${enable_exporting_mongodb}])

1478 +AM_CONDITIONAL([ENABLE_EXPORTING_MONGODB], [test "${enable_exporting_mongodb}" = "yes"])

1479

1480

1481 # -----------------------------------------------------------------------------

@@ -1523,9 +1523,9 @@ AC_MSG_RESULT([${enable_lto}])

1523

1524 # -----------------------------------------------------------------------------

1525

1526 -if test "${enable_backend_kinesis}" = "yes" -o \

1526 +if test "${enable_exporting_kinesis}" = "yes" -o \

1527 "${enable_exporting_pubsub}" = "yes" -o \

1528 - "${enable_backend_prometheus_remote_write}" = "yes" -o \

1528 + "${enable_exporting_prometheus_remote_write}" = "yes" -o \

1529 "${new_cloud_protocol}" = "yes" -o \

1530 "${build_ml}" = "yes"; then

1531 enable_cxx_linker="yes"

@@ -1688,14 +1688,6 @@ AC_DEFINE_UNQUOTED(

1688 AC_CONFIG_FILES([

1689 Makefile

1690 netdata.spec

1691 - backends/graphite/Makefile

1692 - backends/json/Makefile

1693 - backends/Makefile

1694 - backends/opentsdb/Makefile

1695 - backends/prometheus/Makefile

1696 - backends/prometheus/remote_write/Makefile

1697 - backends/aws_kinesis/Makefile

1698 - backends/mongodb/Makefile

1691 collectors/Makefile

1692 collectors/apps.plugin/Makefile

1693 collectors/cgroups.plugin/Makefile

coverity-scan.sh

+3 -3

@@ -175,11 +175,11 @@ OTHER_OPTIONS+=" --enable-jsonc"

175 OTHER_OPTIONS+=" --enable-plugin-nfacct"

176 OTHER_OPTIONS+=" --enable-plugin-freeipmi"

177 OTHER_OPTIONS+=" --enable-plugin-cups"

178 -OTHER_OPTIONS+=" --enable-backend-prometheus-remote-write"

178 +OTHER_OPTIONS+=" --enable-exporting-prometheus-remote-write"

179 # TODO: enable these plugins too

180 #OTHER_OPTIONS+=" --enable-plugin-xenstat"

181 -#OTHER_OPTIONS+=" --enable-backend-kinesis"

182 -#OTHER_OPTIONS+=" --enable-backend-mongodb"

181 +#OTHER_OPTIONS+=" --enable-exporting-kinesis"

182 +#OTHER_OPTIONS+=" --enable-exporting-mongodb"

183

184 FOUND_OPTS="NO"

185 while [ -n "${1}" ]; do

daemon/common.h

+1 -3

@@ -51,9 +51,7 @@

51 // the registry is actually an API feature

52 #include "registry/registry.h"

54 -// backends for archiving the metrics

55 -#include "backends/backends.h"

56 -// the new exporting engine for archiving the metrics

54 +// exporting engine for archiving the metrics

55 #include "exporting/exporting_engine.h"

57 // the netdata API

daemon/config/README.md

+3 -8

@@ -19,10 +19,9 @@ This config file **is not needed by default**. Netdata works fine out of the box

19 settings.

20 4. `[health]` to [configure](#health-section-options) general settings for [health monitoring](/health/README.md)

21 5. `[registry]` for the [Netdata registry](/registry/README.md).

22 -6. `[backend]` to set up [streaming and replication](/streaming/README.md) options.

23 -7. `[statsd]` for the general settings of the [stats.d.plugin](/collectors/statsd.plugin/README.md).

24 -8. `[plugin:NAME]` sections for each collector plugin, under the comment [Per plugin configuration](#per-plugin-configuration).

25 -9. `[CHART_NAME]` sections for each chart defined, under the comment [Per chart configuration](#per-chart-configuration).

22 +6. `[statsd]` for the general settings of the [stats.d.plugin](/collectors/statsd.plugin/README.md).

23 +7. `[plugin:NAME]` sections for each collector plugin, under the comment [Per plugin configuration](#per-plugin-configuration).

24 +8. `[CHART_NAME]` sections for each chart defined, under the comment [Per chart configuration](#per-chart-configuration).

26 The configuration file is a `name = value` dictionary. Netdata will not complain if you set options unknown to it. When you check the running configuration by accessing the URL `/netdata.conf` on your Netdata server, Netdata will add a comment on settings it does not currently use.

@@ -129,10 +128,6 @@ monitoring](/health/README.md).

128

129 To understand what this section is and how it should be configured, please refer to the [registry documentation](/registry/README.md).

130

132 -### [backend]

133 -

134 -Refer to the [streaming and replication](/streaming/README.md) documentation.

135 -

131 ## Per-plugin configuration

132

133 The configuration options for plugins appear in sections following the pattern `[plugin:NAME]`.

daemon/main.c

-3

@@ -451,9 +451,6 @@ static void backwards_compatible_config() {

451

452 config_move(CONFIG_SECTION_GLOBAL, "web files group",

453 CONFIG_SECTION_WEB, "web files group");

454 -

455 - config_move(CONFIG_SECTION_BACKEND, "opentsdb host tags",

456 - CONFIG_SECTION_BACKEND, "host tags");

454 }

455

456 static void get_netdata_configured_variables() {

daemon/static_threads.c

-9

@@ -85,15 +85,6 @@ const struct netdata_static_thread static_threads_common[] = {

85 .init_routine = NULL,

86 .start_routine = statsd_main

87 },

88 - {

89 - .name = "BACKENDS",

90 - .config_section = NULL,

91 - .config_name = NULL,

92 - .enabled = 1,

93 - .thread = NULL,

94 - .init_routine = NULL,

95 - .start_routine = backends_main

96 - },

88 {

89 .name = "EXPORTING",

90 .config_section = NULL,

daemon/unit_test.c

+1 -1

@@ -1521,7 +1521,7 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name)

1521 , netdata_configured_timezone

1522 , netdata_configured_abbrev_timezone

1523 , netdata_configured_utc_offset

1524 - , config_get(CONFIG_SECTION_BACKEND, "host tags", "")

1524 + , ""

1525 , program_name

1526 , program_version

1527 , default_rrd_update_every

database/rrd.h

+5 -7

@@ -461,23 +461,21 @@ typedef enum rrdset_flags {

461 // (the master data set should be the one that has the same family and is not detail)

462 RRDSET_FLAG_DEBUG = 1 << 2, // enables or disables debugging for a chart

463 RRDSET_FLAG_OBSOLETE = 1 << 3, // this is marked by the collector/module as obsolete

464 - RRDSET_FLAG_EXPORTING_SEND = 1 << 4, // if set, this chart should be sent to Prometheus web API

465 - RRDSET_FLAG_EXPORTING_IGNORE = 1 << 5, // if set, this chart should not be sent to Prometheus web API

464 + RRDSET_FLAG_EXPORTING_SEND = 1 << 4, // if set, this chart should be sent to Prometheus web API and external databases

465 + RRDSET_FLAG_EXPORTING_IGNORE = 1 << 5, // if set, this chart should not be sent to Prometheus web API and external databases

466 RRDSET_FLAG_UPSTREAM_SEND = 1 << 6, // if set, this chart should be sent upstream (streaming)

467 RRDSET_FLAG_UPSTREAM_IGNORE = 1 << 7, // if set, this chart should not be sent upstream (streaming)

468 RRDSET_FLAG_UPSTREAM_EXPOSED = 1 << 8, // if set, we have sent this chart definition to netdata parent (streaming)

469 RRDSET_FLAG_STORE_FIRST = 1 << 9, // if set, do not eliminate the first collection during interpolation

470 RRDSET_FLAG_HETEROGENEOUS = 1 << 10, // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)

471 RRDSET_FLAG_HOMOGENEOUS_CHECK = 1 << 11, // if set, the chart should be checked to determine if the dimensions are homogeneous

472 - RRDSET_FLAG_HIDDEN = 1 << 12, // if set, do not show this chart on the dashboard, but use it for backends

472 + RRDSET_FLAG_HIDDEN = 1 << 12, // if set, do not show this chart on the dashboard, but use it for exporting

473 RRDSET_FLAG_SYNC_CLOCK = 1 << 13, // if set, microseconds on next data collection will be ignored (the chart will be synced to now)

474 RRDSET_FLAG_OBSOLETE_DIMENSIONS = 1 << 14, // this is marked by the collector/module when a chart has obsolete dimensions

475 // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at

476 // least rrdset_free_obsolete_time seconds ago.

477 RRDSET_FLAG_ARCHIVED = 1 << 15,

478 RRDSET_FLAG_ACLK = 1 << 16,

479 - RRDSET_FLAG_BACKEND_SEND = 1 << 17, // if set, this chart should be sent to backends

480 - RRDSET_FLAG_BACKEND_IGNORE = 1 << 18 // if set, this chart should not be sent to backends

479 } RRDSET_FLAGS;

480

481 #ifdef HAVE_C___ATOMIC

@@ -632,8 +630,8 @@ typedef enum rrdhost_flags {

630 RRDHOST_FLAG_ORPHAN = 1 << 0, // this host is orphan (not receiving data)

631 RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = 1 << 1, // delete files of obsolete charts

632 RRDHOST_FLAG_DELETE_ORPHAN_HOST = 1 << 2, // delete the entire host when orphan

635 - RRDHOST_FLAG_BACKEND_SEND = 1 << 3, // send it to backends

636 - RRDHOST_FLAG_BACKEND_DONT_SEND = 1 << 4, // don't send it to backends

633 + RRDHOST_FLAG_EXPORTING_SEND = 1 << 3, // send it to external databases

634 + RRDHOST_FLAG_EXPORTING_DONT_SEND = 1 << 4, // don't send it to external databases

635 RRDHOST_FLAG_ARCHIVED = 1 << 5, // The host is archived, no collected charts yet

636 RRDHOST_FLAG_MULTIHOST = 1 << 6, // Host belongs to localhost/megadb

637 } RRDHOST_FLAGS;

database/rrdhost.c

+1 -47

@@ -718,7 +718,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {

718 , netdata_configured_timezone

719 , netdata_configured_abbrev_timezone

720 , netdata_configured_utc_offset

721 - , config_get(CONFIG_SECTION_BACKEND, "host tags", "")

721 + , ""

722 , program_name

723 , program_version

724 , default_rrd_update_every

@@ -1233,50 +1233,6 @@ struct label *parse_json_tags(struct label *label_list, const char *tags)

1233 return label_list;

1234 }

1235

1236 -static struct label *rrdhost_load_labels_from_tags(void)

1237 -{

1238 - if (!localhost->tags)

1239 - return NULL;

1240 -

1241 - struct label *label_list = NULL;

1242 - BACKEND_TYPE type = BACKEND_TYPE_UNKNOWN;

1243 -

1244 - if (config_exists(CONFIG_SECTION_BACKEND, "enabled")) {

1245 - if (config_get_boolean(CONFIG_SECTION_BACKEND, "enabled", CONFIG_BOOLEAN_NO) != CONFIG_BOOLEAN_NO) {

1246 - const char *type_name = config_get(CONFIG_SECTION_BACKEND, "type", "graphite");

1247 - type = backend_select_type(type_name);

1248 - }

1249 - }

1250 -

1251 - switch (type) {

1252 - case BACKEND_TYPE_GRAPHITE:

1253 - label_list = parse_simple_tags(

1254 - label_list, localhost->tags, '=', ';', DO_NOT_STRIP_QUOTES, DO_NOT_STRIP_QUOTES,

1255 - DO_NOT_SKIP_ESCAPED_CHARACTERS);

1256 - break;

1257 - case BACKEND_TYPE_OPENTSDB_USING_TELNET:

1258 - label_list = parse_simple_tags(

1259 - label_list, localhost->tags, '=', ' ', DO_NOT_STRIP_QUOTES, DO_NOT_STRIP_QUOTES,

1260 - DO_NOT_SKIP_ESCAPED_CHARACTERS);

1261 - break;

1262 - case BACKEND_TYPE_OPENTSDB_USING_HTTP:

1263 - label_list = parse_simple_tags(

1264 - label_list, localhost->tags, ':', ',', STRIP_QUOTES, STRIP_QUOTES,

1265 - DO_NOT_SKIP_ESCAPED_CHARACTERS);

1266 - break;

1267 - case BACKEND_TYPE_JSON:

1268 - label_list = parse_json_tags(label_list, localhost->tags);

1269 - break;

1270 - default:

1271 - label_list = parse_simple_tags(

1272 - label_list, localhost->tags, '=', ',', DO_NOT_STRIP_QUOTES, STRIP_QUOTES,

1273 - DO_NOT_SKIP_ESCAPED_CHARACTERS);

1274 - break;

1275 - }

1276 -

1277 - return label_list;

1278 -}

1279 -

1236 static struct label *rrdhost_load_kubernetes_labels(void)

1237 {

1238 struct label *l=NULL;

@@ -1340,10 +1296,8 @@ void reload_host_labels(void)

1296 struct label *from_auto = rrdhost_load_auto_labels();

1297 struct label *from_k8s = rrdhost_load_kubernetes_labels();

1298 struct label *from_config = rrdhost_load_config_labels();

1343 - struct label *from_tags = rrdhost_load_labels_from_tags();

1299

1300 struct label *new_labels = merge_label_lists(from_auto, from_k8s);

1346 - new_labels = merge_label_lists(new_labels, from_tags);

1301 new_labels = merge_label_lists(new_labels, from_config);

1302

1303 rrdhost_rdlock(localhost);

database/rrdset.c

-4

@@ -190,8 +190,6 @@ int rrdset_set_name(RRDSET *st, const char *name) {

190

191 rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND);

192 rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE);

193 - rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_SEND);

194 - rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_IGNORE);

193 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND);

194 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_IGNORE);

195 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);

@@ -872,8 +870,6 @@ RRDSET *rrdset_create_custom(

870 rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);

871 rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_SEND);

872 rrdset_flag_clear(st, RRDSET_FLAG_EXPORTING_IGNORE);

875 - rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_SEND);

876 - rrdset_flag_clear(st, RRDSET_FLAG_BACKEND_IGNORE);

873 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_SEND);

874 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_IGNORE);

875 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);

docs/export/external-databases.md

-1

@@ -85,6 +85,5 @@ Graphite_](/docs/guides/export/export-netdata-metrics-graphite.md).

85 ### Related reference documentation

87 - [Exporting engine reference](/exporting/README.md)

88 -- [Backends reference (deprecated)](/backends/README.md)

docs/guides/using-host-labels.md

+1 -1

@@ -86,7 +86,7 @@ They capture the following:

86 - What virtualization layer the system runs on top of, if any

87 - Whether the system is a streaming parent or child

89 -If you want to organize your systems without manually creating host tags, try the automatic labels in some of the

89 +If you want to organize your systems without manually creating host labels, try the automatic labels in some of the

90 features below.

92 ## Host labels in streaming

exporting/README.md

-12

@@ -20,12 +20,6 @@ the same time. You can have different update intervals and filters configured fo

20 When you enable the exporting engine and a connector, the Netdata Agent exports metrics _beginning from the time you

21 restart its process_, not the entire [database of long-term metrics](/docs/store/change-metrics-storage.md).

23 -The exporting engine has its own configuration file `exporting.conf`. The configuration is almost similar to the

24 -deprecated [backends](/backends/README.md#configuration) system. The most important difference is that type of a

25 -connector should be specified in a section name before a colon and an instance name after the colon. Also, you can't use

26 -`host tags` anymore. Set your labels using the [`[host labels]`](/docs/guides/using-host-labels.md) section in

27 -`netdata.conf`.

28 -

23 Since Netdata collects thousands of metrics per server per second, which would easily congest any database server when

24 several Netdata servers are sending data to it, Netdata allows sending metrics at a lower frequency, by resampling them.

@@ -271,12 +265,6 @@ Configure individual connectors and override any global settings with the follow

265 - `send automatic labels = yes | no` controls if automatically created labels, like `_os_name` or `_architecture`

266 should be sent to the external database

267

274 -> Starting from Netdata v1.20 the host tags (defined in the `[backend]` section of `netdata.conf`) are parsed in

275 -> accordance with a configured backend type and stored as host labels so that they can be reused in API responses and

276 -> exporting connectors. The parsing is supported for graphite, json, opentsdb, and prometheus (default) backend types.

277 -> You can check how the host tags were parsed using the /api/v1/info API call. But, keep in mind that backends subsystem

278 -> is deprecated and will be deleted soon. Please move your existing tags to the `[host labels]` section.

279 -

268 ## HTTPS

269

270 Netdata can send metrics to external databases using the TLS/SSL protocol. Unfortunately, some of

exporting/TIMESCALE.md

+1 -1

@@ -20,7 +20,7 @@ What's TimescaleDB? Here's how their team defines the project on their [GitHub p

21 To get started archiving metrics to TimescaleDB right away, check out Mahlon's [`netdata-timescale-relay`

22 repository](https://github.com/mahlonsmith/netdata-timescale-relay) on GitHub. Please be aware that backends subsystem

23 -is deprecated and Netdata configuration should be moved to the new `exporting conf` configuration file. Use

23 +was removed and Netdata configuration should be moved to the new `exporting.conf` configuration file. Use

24 ```conf

25 [json:my_instance]

26 ```

exporting/aws_kinesis/aws_kinesis.c

+2 -2

@@ -151,7 +151,7 @@ void aws_kinesis_connector_worker(void *instance_p)

151 char error_message[ERROR_LINE_MAX + 1] = "";

152

153 debug(

154 - D_BACKEND,

154 + D_EXPORTING,

155 "EXPORTING: kinesis_put_record(): dest = %s, id = %s, key = %s, stream = %s, partition_key = %s, \

156 buffer = %zu, record = %zu",

157 instance->config.destination,

@@ -175,7 +175,7 @@ void aws_kinesis_connector_worker(void *instance_p)

175 // oops! we couldn't send (all or some of the) data

176 error("EXPORTING: %s", error_message);

177 error(

178 - "EXPORTING: failed to write data to database backend '%s'. Willing to write %zu bytes, wrote %zu bytes.",

178 + "EXPORTING: failed to write data to external database '%s'. Willing to write %zu bytes, wrote %zu bytes.",

179 instance->config.destination, sent_bytes, sent_bytes - lost_bytes);

180

181 stats->transmission_failures++;

exporting/check_filters.c

+7 -7

@@ -16,19 +16,19 @@ int rrdhost_is_exportable(struct instance *instance, RRDHOST *host)

17 RRDHOST_FLAGS *flags = &host->exporting_flags[instance->index];

19 - if (unlikely((*flags & (RRDHOST_FLAG_BACKEND_SEND | RRDHOST_FLAG_BACKEND_DONT_SEND)) == 0)) {

19 + if (unlikely((*flags & (RRDHOST_FLAG_EXPORTING_SEND | RRDHOST_FLAG_EXPORTING_DONT_SEND)) == 0)) {

20 char *host_name = (host == localhost) ? "localhost" : host->hostname;

22 if (!instance->config.hosts_pattern || simple_pattern_matches(instance->config.hosts_pattern, host_name)) {

23 - *flags |= RRDHOST_FLAG_BACKEND_SEND;

23 + *flags |= RRDHOST_FLAG_EXPORTING_SEND;

24 info("enabled exporting of host '%s' for instance '%s'", host_name, instance->config.name);

25 } else {

26 - *flags |= RRDHOST_FLAG_BACKEND_DONT_SEND;

26 + *flags |= RRDHOST_FLAG_EXPORTING_DONT_SEND;

27 info("disabled exporting of host '%s' for instance '%s'", host_name, instance->config.name);

28 }

29 }

31 - if (likely(*flags & RRDHOST_FLAG_BACKEND_SEND))

31 + if (likely(*flags & RRDHOST_FLAG_EXPORTING_SEND))

32 return 1;

33 else

34 return 0;

@@ -65,18 +65,18 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st)

65 *flags |= RRDSET_FLAG_EXPORTING_SEND;

66 else {

67 *flags |= RRDSET_FLAG_EXPORTING_IGNORE;

68 - debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is disabled for backends.", st->id, host->hostname);

68 + debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.", st->id, host->hostname);

69 return 0;

70 }

71 }

73 if(unlikely(!rrdset_is_available_for_exporting_and_alarms(st))) {

74 - debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s', because it is not available for backends.", st->id, host->hostname);

74 + debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.", st->id, host->hostname);

75 return 0;

76 }

78 if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {

79 - debug(D_BACKEND, "BACKEND: not sending chart '%s' of host '%s' because its memory mode is '%s' and the backend requires database access.", st->id, host->hostname, rrd_memory_mode_name(host->rrd_memory_mode));

79 + debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", st->id, host->hostname, rrd_memory_mode_name(host->rrd_memory_mode));

80 return 0;

81 }

exporting/exporting_engine.h

+7 -2

@@ -34,6 +34,9 @@ typedef enum exporting_options {

34 (EXPORTING_SOURCE_DATA_AS_COLLECTED | EXPORTING_SOURCE_DATA_AVERAGE | EXPORTING_SOURCE_DATA_SUM)

35 #define EXPORTING_OPTIONS_DATA_SOURCE(exporting_options) (exporting_options & EXPORTING_OPTIONS_SOURCE_BITS)

37 +extern EXPORTING_OPTIONS global_exporting_options;

38 +extern const char *global_exporting_prefix;

39 +

40 #define sending_labels_configured(instance) \

41 (instance->config.options & (EXPORTING_OPTION_SEND_CONFIGURED_LABELS | EXPORTING_OPTION_SEND_AUTOMATIC_LABELS))

@@ -51,11 +54,11 @@ typedef enum exporting_connector_types {

54 EXPORTING_CONNECTOR_TYPE_JSON_HTTP, // Send data in JSON format using HTTP API

55 EXPORTING_CONNECTOR_TYPE_OPENTSDB, // Send data to OpenTSDB using telnet API

56 EXPORTING_CONNECTOR_TYPE_OPENTSDB_HTTP, // Send data to OpenTSDB using HTTP API

54 - EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE, // User selected to use Prometheus backend

57 + EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE, // Send data using Prometheus remote write protocol

58 EXPORTING_CONNECTOR_TYPE_KINESIS, // Send message to AWS Kinesis

59 EXPORTING_CONNECTOR_TYPE_PUBSUB, // Send message to Google Cloud Pub/Sub

60 EXPORTING_CONNECTOR_TYPE_MONGODB, // Send data to MongoDB collection

58 - EXPORTING_CONNECTOR_TYPE_NUM // Number of backend types

61 + EXPORTING_CONNECTOR_TYPE_NUM // Number of exporting connector types

62 } EXPORTING_CONNECTOR_TYPE;

64 struct engine;

@@ -265,6 +268,8 @@ size_t exporting_name_copy(char *dst, const char *src, size_t max_len);

268 int rrdhost_is_exportable(struct instance *instance, RRDHOST *host);

269 int rrdset_is_exportable(struct instance *instance, RRDSET *st);

270

271 +extern EXPORTING_OPTIONS exporting_parse_data_source(const char *source, EXPORTING_OPTIONS exporting_options);

272 +

273 calculated_number exporting_calculate_value_from_stored_data(

274 struct instance *instance,

275 RRDDIM *rd,

exporting/mongodb/mongodb.c

+1 -1

@@ -327,7 +327,7 @@ void mongodb_connector_worker(void *instance_p)

327 }

328

329 debug(

330 - D_BACKEND,

330 + D_EXPORTING,

331 "EXPORTING: mongodb_insert(): destination = %s, database = %s, collection = %s, data size = %zu",

332 instance->config.destination,

333 connector_specific_config->database,

exporting/nc-exporting.sh

+22 -22

@@ -2,17 +2,17 @@

3 # SPDX-License-Identifier: GPL-3.0-or-later

5 -# This is a simple backend database proxy, written in BASH, using the nc command.

5 +# This is a simple exporting proxy, written in BASH, using the nc command.

6 # Run the script without any parameters for help.

8 MODE="${1}"

9 MY_PORT="${2}"

10 -BACKEND_HOST="${3}"

11 -BACKEND_PORT="${4}"

12 -FILE="${NETDATA_NC_BACKEND_DIR-/tmp}/netdata-nc-backend-${MY_PORT}"

10 +EXPORTING_HOST="${3}"

11 +EXPORTING_PORT="${4}"

12 +FILE="${NETDATA_NC_EXPORTING_DIR-/tmp}/netdata-nc-exporting-${MY_PORT}"

14 log() {

15 - logger --stderr --id=$$ --tag "netdata-nc-backend" "${*}"

15 + logger --stderr --id=$$ --tag "netdata-nc-exporting" "${*}"

16 }

18 mync() {

@@ -28,7 +28,7 @@ mync() {

28 }

30 listen_save_replay_forever() {

31 - local file="${1}" port="${2}" real_backend_host="${3}" real_backend_port="${4}" ret delay=1 started ended

31 + local file="${1}" port="${2}" real_exporting_host="${3}" real_exporting_port="${4}" ret delay=1 started ended

33 while true

34 do

@@ -40,23 +40,23 @@ listen_save_replay_forever() {

41 if [ -s "${file}" ]

42 then

43 - if [ -n "${real_backend_host}" ] && [ -n "${real_backend_port}" ]

43 + if [ -n "${real_exporting_host}" ] && [ -n "${real_exporting_port}" ]

44 then

45 - log "Attempting to send the metrics to the real backend at ${real_backend_host}:${real_backend_port}"

45 + log "Attempting to send the metrics to the real external database at ${real_exporting_host}:${real_exporting_port}"

47 - mync "${real_backend_host}" "${real_backend_port}" <"${file}"

47 + mync "${real_exporting_host}" "${real_exporting_port}" <"${file}"

48 ret=$?

50 if [ ${ret} -eq 0 ]

51 then

52 - log "Successfully sent the metrics to ${real_backend_host}:${real_backend_port}"

52 + log "Successfully sent the metrics to ${real_exporting_host}:${real_exporting_port}"

53 mv "${file}" "${file}.old"

54 touch "${file}"

55 else

56 - log "Failed to send the metrics to ${real_backend_host}:${real_backend_port} (nc returned ${ret}) - appending more data to ${file}"

56 + log "Failed to send the metrics to ${real_exporting_host}:${real_exporting_port} (nc returned ${ret}) - appending more data to ${file}"

57 fi

58 else

59 - log "No backend configured - appending more data to ${file}"

59 + log "No external database configured - appending more data to ${file}"

60 fi

61 fi

@@ -92,7 +92,7 @@ if [ "${MODE}" = "start" ]

92 # save our PID to the lock file

93 echo "$$" >"${FILE}.lock"

95 - listen_save_replay_forever "${FILE}" "${MY_PORT}" "${BACKEND_HOST}" "${BACKEND_PORT}"

95 + listen_save_replay_forever "${FILE}" "${MY_PORT}" "${EXPORTING_HOST}" "${EXPORTING_PORT}"

96 ret=$?

98 log "listener exited."

@@ -131,20 +131,20 @@ else

131 cat <<EOF

132 Usage:

133

134 - "${0}" start|stop PORT [BACKEND_HOST BACKEND_PORT]

134 + "${0}" start|stop PORT [EXPORTING_HOST EXPORTING_PORT]

135

136 PORT The port this script will listen

137 - (configure netdata to use this as a second backend)

137 + (configure netdata to use this as an external database)

138

139 - BACKEND_HOST The real backend host

140 - BACKEND_PORT The real backend port

139 + EXPORTING_HOST The real host for the external database

140 + EXPORTING_PORT The real port for the external database

141

142 - This script can act as fallback backend for netdata.

142 + This script can act as fallback database for netdata.

143 It will receive metrics from netdata, save them to

144 ${FILE}

145 - and once netdata reconnects to the real-backend, this script

146 - will push all metrics collected to the real-backend too and

147 - wait for a failure to happen again.

145 + and once netdata reconnects to the real external database,

146 + this script will push all metrics collected to the real

147 + external database too and wait for a failure to happen again.

148

149 Only one netdata can connect to this script at a time.

150 If you need fallback for multiple netdata, run this script

@@ -152,7 +152,7 @@ Usage:

152

153 You can run me in the background with this:

154

155 - screen -d -m "${0}" start PORT [BACKEND_HOST BACKEND_PORT]

155 + screen -d -m "${0}" start PORT [EXPORTING_HOST EXPORTING_PORT]

156 EOF

157 exit 1

158 fi

exporting/process_data.c

+2 -2

@@ -109,7 +109,7 @@ calculated_number exporting_calculate_value_from_stored_data(

109 if (unlikely(before < first_t || after > last_t)) {

110 // the chart has not been updated in the wanted timeframe

111 debug(

112 - D_BACKEND,

112 + D_EXPORTING,

113 "EXPORTING: %s.%s.%s: aligned timeframe %lu to %lu is outside the chart's database range %lu to %lu",

114 host->hostname,

115 st->id,

@@ -143,7 +143,7 @@ calculated_number exporting_calculate_value_from_stored_data(

143 rd->state->query_ops.finalize(&handle);

144 if (unlikely(!counter)) {

145 debug(

146 - D_BACKEND,

146 + D_EXPORTING,

147 "EXPORTING: %s.%s.%s: no values stored in database for range %lu to %lu",

148 host->hostname,

149 st->id,

exporting/prometheus/prometheus.c

+3 -3

@@ -35,7 +35,7 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st)

35 else {

36 rrdset_flag_set(st, RRDSET_FLAG_EXPORTING_IGNORE);

37 debug(

38 - D_BACKEND,

38 + D_EXPORTING,

39 "EXPORTING: not sending chart '%s' of host '%s', because it is disabled for exporting.",

40 st->id,

41 host->hostname);

@@ -45,7 +45,7 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st)

46 if (unlikely(!rrdset_is_available_for_exporting_and_alarms(st))) {

47 debug(

48 - D_BACKEND,

48 + D_EXPORTING,

49 "EXPORTING: not sending chart '%s' of host '%s', because it is not available for exporting.",

50 st->id,

51 host->hostname);

@@ -56,7 +56,7 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st)

56 st->rrd_memory_mode == RRD_MEMORY_MODE_NONE &&

57 !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {

58 debug(

59 - D_BACKEND,

59 + D_EXPORTING,

60 "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting connector requires database access.",

61 st->id,

62 host->hostname,

exporting/prometheus/remote_write/remote_write.c

+1 -1

@@ -236,7 +236,7 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *

236

237 if (unlikely(rd->last_collected_time.tv_sec < instance->after)) {

238 debug(

239 - D_BACKEND,

239 + D_EXPORTING,

240 "EXPORTING: not sending dimension '%s' of chart '%s' from host '%s', "

241 "its last data collection (%lu) is not within our timeframe (%lu to %lu)",

242 rd->id, rd->rrdset->id,

exporting/pubsub/pubsub.c

+1 -1

@@ -141,7 +141,7 @@ void pubsub_connector_worker(void *instance_p)

141 }

142

143 debug(

144 - D_BACKEND, "EXPORTING: pubsub_publish(): project = %s, topic = %s, buffer = %zu",

144 + D_EXPORTING, "EXPORTING: pubsub_publish(): project = %s, topic = %s, buffer = %zu",

145 connector_specific_config->project_id, connector_specific_config->topic_id, buffer_len);

146

147 if (pubsub_publish((void *)connector_specific_data, error_message, stats->buffered_metrics, buffer_len)) {

exporting/read_config.c

+17 -22

@@ -2,6 +2,9 @@

3 #include "exporting_engine.h"

5 +EXPORTING_OPTIONS global_exporting_options = EXPORTING_SOURCE_DATA_AVERAGE | EXPORTING_OPTION_SEND_NAMES;

6 +const char *global_exporting_prefix = "netdata";

7 +

8 struct config exporting_config = { .first_section = NULL,

9 .last_section = NULL,

10 .mutex = NETDATA_MUTEX_INITIALIZER,

@@ -160,7 +163,7 @@ EXPORTING_CONNECTOR_TYPE exporting_select_type(const char *type)

163 return EXPORTING_CONNECTOR_TYPE_UNKNOWN;

164 }

165

163 -EXPORTING_OPTIONS exporting_parse_data_source(const char *data_source, EXPORTING_OPTIONS exporting_options)

166 +inline EXPORTING_OPTIONS exporting_parse_data_source(const char *data_source, EXPORTING_OPTIONS exporting_options)

167 {

168 if (!strcmp(data_source, "raw") || !strcmp(data_source, "as collected") || !strcmp(data_source, "as-collected") ||

169 !strcmp(data_source, "as_collected") || !strcmp(data_source, "ascollected")) {

@@ -194,7 +197,7 @@ struct engine *read_exporting_config()

197 static struct engine *engine = NULL;

198 struct connector_instance_list {

199 struct connector_instance local_ci;

197 - EXPORTING_CONNECTOR_TYPE backend_type;

200 + EXPORTING_CONNECTOR_TYPE exporting_type;

201

202 struct connector_instance_list *next;

203 };

@@ -238,21 +241,14 @@ struct engine *read_exporting_config()

241 prometheus_exporter_instance->config.update_every =

242 prometheus_config_get_number(EXPORTING_UPDATE_EVERY_OPTION_NAME, EXPORTING_UPDATE_EVERY_DEFAULT);

243

241 - // wait for backend subsystem to be initialized

242 - for (int retries = 0; !global_backend_source && retries < 1000; retries++)

243 - sleep_usec(10000);

244 -

245 - if (!global_backend_source)

246 - global_backend_source = "average";

247 -

248 - prometheus_exporter_instance->config.options |= global_backend_options & EXPORTING_OPTIONS_SOURCE_BITS;

244 + prometheus_exporter_instance->config.options |= global_exporting_options & EXPORTING_OPTIONS_SOURCE_BITS;

245

250 - char *data_source = prometheus_config_get("data source", global_backend_source);

246 + char *data_source = prometheus_config_get("data source", "average");

247 prometheus_exporter_instance->config.options =

248 exporting_parse_data_source(data_source, prometheus_exporter_instance->config.options);

249

250 if (prometheus_config_get_boolean(

255 - "send names instead of ids", global_backend_options & EXPORTING_OPTION_SEND_NAMES))

251 + "send names instead of ids", global_exporting_options & EXPORTING_OPTION_SEND_NAMES))

252 prometheus_exporter_instance->config.options |= EXPORTING_OPTION_SEND_NAMES;

253 else

254 prometheus_exporter_instance->config.options &= ~EXPORTING_OPTION_SEND_NAMES;

@@ -268,18 +264,17 @@ struct engine *read_exporting_config()

264 prometheus_exporter_instance->config.options &= ~EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;

265

266 prometheus_exporter_instance->config.charts_pattern = simple_pattern_create(

271 - prometheus_config_get("send charts matching", global_backend_send_charts_matching),

267 + prometheus_config_get("send charts matching", "*"),

268 NULL,

269 SIMPLE_PATTERN_EXACT);

270 prometheus_exporter_instance->config.hosts_pattern = simple_pattern_create(

271 prometheus_config_get("send hosts matching", "localhost *"), NULL, SIMPLE_PATTERN_EXACT);

272

277 - prometheus_exporter_instance->config.prefix = prometheus_config_get("prefix", global_backend_prefix);

273 + prometheus_exporter_instance->config.prefix = prometheus_config_get("prefix", global_exporting_prefix);

274

275 prometheus_exporter_instance->config.initialized = 1;

276 }

277

282 - // TODO: change BACKEND to EXPORTING

278 while (get_connector_instance(&local_ci)) {

279 info("Processing connector instance (%s)", local_ci.instance_name);

280

@@ -290,7 +285,7 @@ struct engine *read_exporting_config()

285

286 tmp_ci_list = (struct connector_instance_list *)callocz(1, sizeof(struct connector_instance_list));

287 memcpy(&tmp_ci_list->local_ci, &local_ci, sizeof(local_ci));

293 - tmp_ci_list->backend_type = exporting_select_type(local_ci.connector_name);

288 + tmp_ci_list->exporting_type = exporting_select_type(local_ci.connector_name);

289 tmp_ci_list->next = tmp_ci_list_prev;

290 tmp_ci_list_prev = tmp_ci_list;

291 instances_to_activate++;

@@ -320,34 +315,34 @@ struct engine *read_exporting_config()

315

316 info("Instance %s on %s", tmp_ci_list->local_ci.instance_name, tmp_ci_list->local_ci.connector_name);

317

323 - if (tmp_ci_list->backend_type == EXPORTING_CONNECTOR_TYPE_UNKNOWN) {

318 + if (tmp_ci_list->exporting_type == EXPORTING_CONNECTOR_TYPE_UNKNOWN) {

319 error("Unknown exporting connector type");

320 goto next_connector_instance;

321 }

322

323 #ifndef ENABLE_PROMETHEUS_REMOTE_WRITE

329 - if (tmp_ci_list->backend_type == EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE) {

324 + if (tmp_ci_list->exporting_type == EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE) {

325 error("Prometheus Remote Write support isn't compiled");

326 goto next_connector_instance;

327 }

328 #endif

329

330 #ifndef HAVE_KINESIS

336 - if (tmp_ci_list->backend_type == EXPORTING_CONNECTOR_TYPE_KINESIS) {

331 + if (tmp_ci_list->exporting_type == EXPORTING_CONNECTOR_TYPE_KINESIS) {

332 error("AWS Kinesis support isn't compiled");

333 goto next_connector_instance;

334 }

335 #endif

336

337 #ifndef ENABLE_EXPORTING_PUBSUB

343 - if (tmp_ci_list->backend_type == EXPORTING_CONNECTOR_TYPE_PUBSUB) {

338 + if (tmp_ci_list->exporting_type == EXPORTING_CONNECTOR_TYPE_PUBSUB) {

339 error("Google Cloud Pub/Sub support isn't compiled");

340 goto next_connector_instance;

341 }

342 #endif

343

344 #ifndef HAVE_MONGOC

350 - if (tmp_ci_list->backend_type == EXPORTING_CONNECTOR_TYPE_MONGODB) {

345 + if (tmp_ci_list->exporting_type == EXPORTING_CONNECTOR_TYPE_MONGODB) {

346 error("MongoDB support isn't compiled");

347 goto next_connector_instance;

348 }

@@ -358,7 +353,7 @@ struct engine *read_exporting_config()

353 engine->instance_root = tmp_instance;

354

355 tmp_instance->engine = engine;

361 - tmp_instance->config.type = tmp_ci_list->backend_type;

356 + tmp_instance->config.type = tmp_ci_list->exporting_type;

357

358 instance_name = tmp_ci_list->local_ci.instance_name;

359

exporting/send_data.c

+1 -1

@@ -41,7 +41,7 @@ int exporting_discard_response(BUFFER *buffer, struct instance *instance) {

41 *d = '\0';

43 debug(

44 - D_BACKEND,

44 + D_EXPORTING,

45 "EXPORTING: received %zu bytes from %s connector instance. Ignoring them. Sample: '%s'",

46 buffer_strlen(buffer),

47 instance->config.name,

exporting/tests/test_exporting_engine.c

+2 -7

@@ -14,11 +14,6 @@ char *netdata_configured_hostname = "test_global_host";

15 char log_line[MAX_LOG_LINE + 1];

17 -BACKEND_OPTIONS global_backend_options = 0;

18 -const char *global_backend_source = "average";

19 -const char *global_backend_prefix = "netdata";

20 -const char *global_backend_send_charts_matching = "*";

21 -

17 void init_connectors_in_tests(struct engine *engine)

18 {

19 expect_function_call(__wrap_now_realtime_sec);

@@ -235,7 +230,7 @@ static void test_rrdhost_is_exportable(void **state)

230 assert_string_equal(log_line, "enabled exporting of host 'localhost' for instance 'instance_name'");

231

232 assert_ptr_not_equal(localhost->exporting_flags, NULL);

238 - assert_int_equal(localhost->exporting_flags[0], RRDHOST_FLAG_BACKEND_SEND);

233 + assert_int_equal(localhost->exporting_flags[0], RRDHOST_FLAG_EXPORTING_SEND);

234 }

235

236 static void test_false_rrdhost_is_exportable(void **state)

@@ -255,7 +250,7 @@ static void test_false_rrdhost_is_exportable(void **state)

250 assert_string_equal(log_line, "disabled exporting of host 'localhost' for instance 'instance_name'");

251

252 assert_ptr_not_equal(localhost->exporting_flags, NULL);

258 - assert_int_equal(localhost->exporting_flags[0], RRDHOST_FLAG_BACKEND_DONT_SEND);

253 + assert_int_equal(localhost->exporting_flags[0], RRDHOST_FLAG_EXPORTING_DONT_SEND);

254 }

255

256 static void test_rrdset_is_exportable(void **state)

health/Makefile.am

-1

@@ -27,7 +27,6 @@ dist_healthconfig_DATA = \

27 health.d/adaptec_raid.conf \

28 health.d/anomalies.conf \

29 health.d/apcupsd.conf \

30 - health.d/backend.conf \

30 health.d/bcache.conf \

31 health.d/beanstalkd.conf \

32 health.d/bind_rndc.conf \

health/health.d/backend.conf deleted

-42

@@ -1,42 +0,0 @@

1 -# Alert that backends subsystem will be disabled soon

2 - alarm: backend_metrics_eol

3 - on: netdata.backend_metrics

4 - class: Errors

5 - type: Netdata

6 -component: Exporting engine

7 - units: boolean

8 - calc: $now - $last_collected_t

9 - every: 1m

10 - warn: $this > 0

11 - delay: down 5m multiplier 1.5 max 1h

12 - info: the backends subsystem is deprecated and will be removed soon. Migrate your configuration to exporting.conf.

13 - to: sysadmin

14 -

15 -# make sure we are sending data to backend

16 -

17 - alarm: backend_last_buffering

18 - on: netdata.backend_metrics

19 - class: Latency

20 - type: Netdata

21 -component: Exporting engine

22 - calc: $now - $last_collected_t

23 - units: seconds ago

24 - every: 10s

25 - warn: $this > (($status >= $WARNING) ? ($update_every) : ( 5 * $update_every))

26 - crit: $this > (($status == $CRITICAL) ? ($update_every) : (60 * $update_every))

27 - delay: down 5m multiplier 1.5 max 1h

28 - info: number of seconds since the last successful buffering of backend data

29 - to: dba

30 -

31 - alarm: backend_metrics_sent

32 - on: netdata.backend_metrics

33 - class: Workload

34 - type: Netdata

35 -component: Exporting engine

36 - units: %

37 - calc: abs($sent) * 100 / abs($buffered)

38 - every: 10s

39 - warn: $this != 100

40 - delay: down 5m multiplier 1.5 max 1h

41 - info: percentage of metrics sent to the backend server

42 - to: dba

libnetdata/buffer/README.md

+1 -2

@@ -13,5 +13,4 @@ Also, they are super fast in printing and appending data to the string and its `

13 is just a lookup (it does not traverse the string).

15 Netdata uses `BUFFER`s for preparing web responses and buffering data to be sent upstream or

16 -to backend databases.

17 -

16 +to external databases.

libnetdata/config/appconfig.c

+3 -4

@@ -591,7 +591,7 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons

591 int line = 0;

592 struct section *co = NULL;

593 int is_exporter_config = 0;

594 - int _backends = 0; // number of backend sections we have

594 + int _connectors = 0; // number of exporting connector sections we have

595 char working_instance[CONFIG_MAX_NAME + 1];

596 char working_connector[CONFIG_MAX_NAME + 1];

597 struct section *working_connector_section = NULL;

@@ -641,8 +641,8 @@ int appconfig_load(struct config *root, char *filename, int overwrite_used, cons

641 strncpy(working_connector, s, CONFIG_MAX_NAME);

642 s = s + rc + 1;

643 if (unlikely(!(*s))) {

644 - _backends++;

645 - sprintf(buffer, "instance_%d", _backends);

644 + _connectors++;

645 + sprintf(buffer, "instance_%d", _connectors);

646 s = buffer;

647 }

648 strncpy(working_instance, s, CONFIG_MAX_NAME);

@@ -793,7 +793,6 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)

793 || !strcmp(co->name, CONFIG_SECTION_CLOUD)

794 || !strcmp(co->name, CONFIG_SECTION_REGISTRY)

795 || !strcmp(co->name, CONFIG_SECTION_HEALTH)

796 - || !strcmp(co->name, CONFIG_SECTION_BACKEND)

796 || !strcmp(co->name, CONFIG_SECTION_STREAM)

797 || !strcmp(co->name, CONFIG_SECTION_HOST_LABEL)

798 || !strcmp(co->name, CONFIG_SECTION_ML)

libnetdata/config/appconfig.h

-1

@@ -89,7 +89,6 @@

89 #define CONFIG_SECTION_CLOUD "cloud"

90 #define CONFIG_SECTION_REGISTRY "registry"

91 #define CONFIG_SECTION_HEALTH "health"

92 -#define CONFIG_SECTION_BACKEND "backend"

92 #define CONFIG_SECTION_STREAM "stream"

93 #define CONFIG_SECTION_ML "ml"

94 #define CONFIG_SECTION_EXPORTING "exporting:global"

libnetdata/log/log.h

+1 -1

@@ -36,7 +36,7 @@ extern "C" {

36 #define D_CONNECT_TO 0x0000000001000000

37 #define D_RRDHOST 0x0000000002000000

38 #define D_LOCKS 0x0000000004000000

39 -#define D_BACKEND 0x0000000008000000

39 +#define D_EXPORTING 0x0000000008000000

40 #define D_STATSD 0x0000000010000000

41 #define D_POLLFD 0x0000000020000000

42 #define D_STREAM 0x0000000040000000

netdata-installer.sh

+20 -15

@@ -245,14 +245,14 @@ USAGE: ${PROGRAM} [options]

245 --disable-plugin-nfacct Explicitly disable the nfacct plugin.

246 --enable-plugin-xenstat Enable the xenstat plugin. Default: enable it when libxenstat and libyajl are available.

247 --disable-plugin-xenstat Explicitly disable the xenstat plugin.

248 - --enable-backend-kinesis Enable AWS Kinesis backend. Default: enable it when libaws_cpp_sdk_kinesis and its dependencies

249 - are available.

250 - --disable-backend-kinesis Explicitly disable AWS Kinesis backend.

251 - --enable-backend-prometheus-remote-write Enable Prometheus remote write backend. Default: enable it when libprotobuf and

252 - libsnappy are available.

253 - --disable-backend-prometheus-remote-write Explicitly disable Prometheus remote write backend.

254 - --enable-backend-mongodb Enable MongoDB backend. Default: enable it when libmongoc is available.

255 - --disable-backend-mongodb Explicitly disable MongoDB backend.

248 + --enable-exporting-kinesis Enable AWS Kinesis exporting connector. Default: enable it when libaws_cpp_sdk_kinesis

249 + and its dependencies are available.

250 + --disable-exporting-kinesis Explicitly disable AWS Kinesis exporting connector.

251 + --enable-exporting-prometheus-remote-write Enable Prometheus remote write exporting connector. Default: enable it

252 + when libprotobuf and libsnappy are available.

253 + --disable-exporting-prometheus-remote-write Explicitly disable Prometheus remote write exporting connector.

254 + --enable-exporting-mongodb Enable MongoDB exporting connector. Default: enable it when libmongoc is available.

255 + --disable-exporting-mongodb Explicitly disable MongoDB exporting connector.

256 --enable-exporting-pubsub Enable Google Cloud PubSub exporting connector. Default: enable it when

257 libgoogle_cloud_cpp_pubsub_protos and its dependencies are available.

258 --disable-exporting-pubsub Explicitly disable Google Cloud PubSub exporting connector.

@@ -345,15 +345,20 @@ while [ -n "${1}" ]; do

345 "--disable-plugin-nfacct") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-plugin-nfacct)}" | sed 's/$/ --disable-plugin-nfacct/g')" ;;

346 "--enable-plugin-xenstat") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-plugin-xenstat)}" | sed 's/$/ --enable-plugin-xenstat/g')" ;;

347 "--disable-plugin-xenstat") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-plugin-xenstat)}" | sed 's/$/ --disable-plugin-xenstat/g')" ;;

348 - "--enable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-backend-kinesis)}" | sed 's/$/ --enable-backend-kinesis/g')" ;;

349 - "--disable-backend-kinesis") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-backend-kinesis)}" | sed 's/$/ --disable-backend-kinesis/g')" ;;

350 - "--enable-backend-prometheus-remote-write") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-backend-prometheus-remote-write)}" | sed 's/$/ --enable-backend-prometheus-remote-write/g')" ;;

351 - "--disable-backend-prometheus-remote-write")

352 - NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-backend-prometheus-remote-write)}" | sed 's/$/ --disable-backend-prometheus-remote-write/g')"

348 + "--enable-exporting-kinesis" | "--enable-backend-kinesis")

349 + NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-exporting-kinesis)}" | sed 's/$/ --enable-exporting-kinesis/g')" ;;

350 + "--disable-exporting-kinesis" | "--disable-backend-kinesis")

351 + NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-exporting-kinesis)}" | sed 's/$/ --disable-exporting-kinesis/g')" ;;

352 + "--enable-exporting-prometheus-remote-write" | "--enable-backend-prometheus-remote-write")

353 + NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-exporting-prometheus-remote-write)}" | sed 's/$/ --enable-exporting-prometheus-remote-write/g')" ;;

354 + "--disable-exporting-prometheus-remote-write" | "--disable-backend-prometheus-remote-write")

355 + NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-exporting-prometheus-remote-write)}" | sed 's/$/ --disable-exporting-prometheus-remote-write/g')"

356 NETDATA_DISABLE_PROMETHEUS=1

357 ;;

355 - "--enable-backend-mongodb") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-backend-mongodb)}" | sed 's/$/ --enable-backend-mongodb/g')" ;;

356 - "--disable-backend-mongodb") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-backend-mongodb)}" | sed 's/$/ --disable-backend-mongodb/g')" ;;

358 + "--enable-exporting-mongodb" | "--enable-backend-mongodb")

359 + NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-exporting-mongodb)}" | sed 's/$/ --enable-exporting-mongodb/g')" ;;

360 + "--disable-exporting-mongodb" | "--disable-backend-mongodb")

361 + NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-exporting-mongodb)}" | sed 's/$/ --disable-exporting-mongodb/g')" ;;

362 "--enable-exporting-pubsub") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-exporting-pubsub)}" | sed 's/$/ --enable-exporting-pubsub/g')" ;;

363 "--disable-exporting-pubsub") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--disable-exporting-pubsub)}" | sed 's/$/ --disable-exporting-pubsub/g')" ;;

364 "--enable-lto") NETDATA_CONFIGURE_OPTIONS="$(echo "${NETDATA_CONFIGURE_OPTIONS%--enable-lto)}" | sed 's/$/ --enable-lto/g')" ;;

packaging/installer/methods/freebsd.md

+5 -6

@@ -93,12 +93,11 @@ The `netdata-updater.sh` script will update your Agent.

93 | `--disable-plugin-nfacct` | Disable nfacct plugin. Default: enable it when libmnl and libnetfilter_acct are available|

94 | `--enable-plugin-xenstat` | Enable the xenstat plugin. Default: enable it when libxenstat and libyajl are available|

95 | `--disable-plugin-xenstat` | Disable the xenstat plugin|

96 -| `--enable-backend-kinesis` | Enable AWS Kinesis backend. Default: enable it when libaws_cpp_sdk_kinesis and libraries (it depends on are available)|

97 -| `--disable-backend-kinesis` | Disable AWS Kinesis backend. Default: enable it when libaws_cpp_sdk_kinesis and libraries (it depends on are available)|

98 -| `--enable-backend-prometheus-remote-write` | Enable Prometheus remote write backend. Default: enable it when libprotobuf and libsnappy are available|

99 -| `--disable-backend-prometheus-remote-write` | Disable Prometheus remote write backend. Default: enable it when libprotobuf and libsnappy are available|

100 -| `--enable-backend-mongodb` | Enable MongoDB backend. Default: enable it when libmongoc is available|

101 -| `--disable-backend-mongodb` | Disable MongoDB backend|

96 +| `--disable-exporting-kinesis` | Disable AWS Kinesis exporting connector. Default: enable it when libaws_cpp_sdk_kinesis and libraries (it depends on are available)|

97 +| `--enable-exporting-prometheus-remote-write` | Enable Prometheus remote write exporting connector. Default: enable it when libprotobuf and libsnappy are available|

98 +| `--disable-exporting-prometheus-remote-write` | Disable Prometheus remote write exporting connector. Default: enable it when libprotobuf and libsnappy are available|

99 +| `--enable-exporting-mongodb` | Enable MongoDB exporting connector. Default: enable it when libmongoc is available|

100 +| `--disable-exporting-mongodb` | Disable MongoDB exporting connector|

101 | `--enable-lto` | Enable Link-Time-Optimization. Default: enabled|

102 | `--disable-lto` | Disable Link-Time-Optimization. Default: enabled|

103 | `--disable-x86-sse` | Disable SSE instructions. By default SSE optimizations are enabled|

streaming/README.md

+6 -14

@@ -67,18 +67,18 @@ Alerts for the child can be triggered by any of the involved hosts that maintain

67 You can daisy-chain any number of Netdata, each with or without a database and

68 with or without alerts for the child metrics.

70 -### Mix and match with backends

70 +### Mix and match with exporting engine

72 -All nodes that maintain a database can also send their data to a backend database.

72 +All nodes that maintain a database can also send their data to an external database.

73 This allows quite complex setups.

75 Example:

77 1. Netdata nodes `A` and `B` do not maintain a database and stream metrics to Netdata node `C`(live streaming functionality).

78 -2. Netdata node `C` maintains a database for `A`, `B`, `C` and archives all metrics to `graphite` with 10 second detail (backends functionality).

78 +2. Netdata node `C` maintains a database for `A`, `B`, `C` and archives all metrics to `graphite` with 10 second detail (exporting functionality).

79 3. Netdata node `C` also streams data for `A`, `B`, `C` to Netdata `D`, which also collects data from `E`, `F` and `G` from another DMZ (live streaming functionality).

80 4. Netdata node `D` is just a proxy, without a database, that streams all data to a remote site at Netdata `H`.

81 -5. Netdata node `H` maintains a database for `A`, `B`, `C`, `D`, `E`, `F`, `G`, `H` and sends all data to `opentsdb` with 5 seconds detail (backends functionality)

81 +5. Netdata node `H` maintains a database for `A`, `B`, `C`, `D`, `E`, `F`, `G`, `H` and sends all data to `opentsdb` with 5 seconds detail (exporting functionality)

82 6. Alerts are triggered by `H` for all hosts.

83 7. Users can use all Netdata nodes that maintain a database to view metrics (i.e. at `H` all hosts can be viewed).

@@ -107,15 +107,7 @@ This also disables the registry (there cannot be a registry without an API).

107 requests from its child nodes. 0 sets no limit, 1 means maximum once every second. If this is set, you may see error log

108 entries "... too busy to accept new streaming request. Will be allowed in X secs".

109

110 -```

111 -[backend]

112 - enabled = yes | no

113 - type = graphite | opentsdb

114 - destination = IP:PORT ...

115 - update every = 10

116 -```

117 -

118 -`[backend]` configures data archiving to a backend (it archives all databases maintained on

110 +You can [use](docs/agent/exporting#configuration) the exporting engine to configure data archiving to an external database (it archives all databases maintained on

111 this host).

112

113 ### Streaming configuration

@@ -156,7 +148,7 @@ a proxy).

148 ```

149 This is an overview of how these options can be combined:

150

152 |------|:-------------:|:----------:|:----------------:|:-----:|:----:|:-------:|

tests/exporting/prometheus-avg-oldunits.txt renamed

tests/exporting/prometheus-avg.txt renamed

tests/exporting/prometheus-raw.txt renamed

tests/exporting/prometheus.bats renamed

+4 -4

@@ -6,14 +6,14 @@ validate_metrics() {

7 curl -sS "http://localhost:19999/api/v1/allmetrics?format=prometheus&prefix=nd&timestamps=no${params}" |

9 - sed -ne 's/{.*//p' | sort | uniq > tests/backends/new-${fname}

10 - diff tests/backends/${fname} tests/backends/new-${fname}

11 - rm tests/backends/new-${fname}

9 + sed -ne 's/{.*//p' | sort | uniq > tests/exportings/new-${fname}

10 + diff tests/exportings/${fname} tests/exportings/new-${fname}

11 + rm tests/exportings/new-${fname}

12 }

15 if [ ! -f .gitignore ]; then

16 - echo "Need to run as ./tests/backends/$(basename "$0") from top level directory of git repository" >&2

16 + echo "Need to run as ./tests/exportings/$(basename "$0") from top level directory of git repository" >&2

17 exit 1

18 fi

web/api/exporters/allmetrics.c

+4 -4

@@ -25,17 +25,17 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client

25 if (prometheus_exporter_instance)

26 prometheus_exporting_options = prometheus_exporter_instance->config.options;

27 else

28 - prometheus_exporting_options = global_backend_options;

28 + prometheus_exporting_options = global_exporting_options;

30 PROMETHEUS_OUTPUT_OPTIONS prometheus_output_options =

31 PROMETHEUS_OUTPUT_TIMESTAMPS |

32 - ((prometheus_exporting_options & BACKEND_OPTION_SEND_NAMES) ? PROMETHEUS_OUTPUT_NAMES : 0);

32 + ((prometheus_exporting_options & EXPORTING_OPTION_SEND_NAMES) ? PROMETHEUS_OUTPUT_NAMES : 0);

34 const char *prometheus_prefix;

35 if (prometheus_exporter_instance)

36 prometheus_prefix = prometheus_exporter_instance->config.prefix;

37 else

38 - prometheus_prefix = global_backend_prefix;

38 + prometheus_prefix = global_exporting_prefix;

40 while(url) {

41 char *value = mystrsep(&url, "&");

@@ -64,7 +64,7 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client

64 prometheus_prefix = value;

65 }

66 else if(!strcmp(name, "data") || !strcmp(name, "source") || !strcmp(name, "data source") || !strcmp(name, "data-source") || !strcmp(name, "data_source") || !strcmp(name, "datasource")) {

67 - prometheus_exporting_options = backend_parse_data_source(value, prometheus_exporting_options);

67 + prometheus_exporting_options = exporting_parse_data_source(value, prometheus_exporting_options);

68 }

69 else {

70 int i;