Add HTTP and HTTPS support to the simple exporting connector (#9911)
Vladimir Kobal committed
Nov 5, 2020 at 19:08 UTC
943ee2482b16a81afd54b426f4fb0952f99c48e7
35 files changed
+846
-398
.gitignore
+2
@@ -146,6 +146,8 @@ cmake-build-release/
146
CMakeCache.txt
147
CMakeFiles/
148
cmake_install.cmake
149
+.cmake
150
+compile_commands.json
151
152
# jetbrains IDE
153
.jetbrains*
CMakeLists.txt
+1
@@ -1270,6 +1270,7 @@ endif()
1270
-Wl,--wrap=connect_to_one_of
1271
-Wl,--wrap=create_main_rusage_chart
1272
-Wl,--wrap=send_main_rusage
1273
+ -Wl,--wrap=simple_connector_end_batch
1274
${PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS}
1275
${KINESIS_LINK_OPTIONS}
1276
${PUBSUB_LINK_OPTIONS}
Makefile.am
+1
@@ -955,6 +955,7 @@ if ENABLE_UNITTESTS
955
-Wl,--wrap=connect_to_one_of \
956
-Wl,--wrap=create_main_rusage_chart \
957
-Wl,--wrap=send_main_rusage \
958
+ -Wl,--wrap=simple_connector_end_batch \
959
$(TEST_LDFLAGS) \
960
$(NULL)
961
exporting_tests_exporting_engine_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
backends/backends.c
+4
-4
@@ -551,7 +551,7 @@ void *backends_main(void *ptr) {
551
case BACKEND_TYPE_OPENTSDB_USING_HTTP: {
552
#ifdef ENABLE_HTTPS
553
if (!strcmp(type, "opentsdb:https")) {
554
- security_start_ssl(NETDATA_SSL_CONTEXT_OPENTSDB);
554
+ security_start_ssl(NETDATA_SSL_CONTEXT_EXPORTING);
555
}
556
#endif
557
backend_set_opentsdb_http_variables(&default_port,&backend_response_checker,&backend_request_formatter);
@@ -1001,9 +1001,9 @@ void *backends_main(void *ptr) {
1001
sock = connect_to_one_of(destination, default_port, &timeout, &reconnects, NULL, 0);
1002
#ifdef ENABLE_HTTPS
1003
if(sock != -1) {
1004
- if(netdata_opentsdb_ctx) {
1004
+ if(netdata_exporting_ctx) {
1005
if(!opentsdb_ssl.conn) {
1006
- opentsdb_ssl.conn = SSL_new(netdata_opentsdb_ctx);
1006
+ opentsdb_ssl.conn = SSL_new(netdata_exporting_ctx);
1007
if(!opentsdb_ssl.conn) {
1008
error("Failed to allocate SSL structure %d.", sock);
1009
opentsdb_ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
@@ -1229,7 +1229,7 @@ cleanup:
1229
buffer_free(response);
1230
1231
#ifdef ENABLE_HTTPS
1232
- if(netdata_opentsdb_ctx) {
1232
+ if(netdata_exporting_ctx) {
1233
if(opentsdb_ssl.conn) {
1234
SSL_free(opentsdb_ssl.conn);
1235
}
exporting/README.md
+12
-3
@@ -44,7 +44,7 @@ X seconds (though, it can send them per second if you need it to).
44
also be configured). Learn more in our guide to [export and visualize Netdata metrics in
45
Graphite](/docs/guides/export/export-netdata-metrics-graphite.md).
46
- [**JSON** document databases](/exporting/json/README.md)
47
- - [**OpenTSDB**](/exporting/opentsdb/README.md): Use a plaintext, HTTP, or HTTPS interfaces. Metrics are sent to
47
+ - [**OpenTSDB**](/exporting/opentsdb/README.md): Use a plaintext or HTTP interfaces. Metrics are sent to
48
OpenTSDB as `prefix.chart.dimension` with tag `host=hostname`.
49
- [**MongoDB**](/exporting/mongodb/README.md): Metrics are sent to the database in `JSON` format.
50
- [**Prometheus**](/exporting/prometheus/README.md): Use an existing Prometheus installation to scrape metrics
@@ -173,8 +173,10 @@ You can configure each connector individually using the available [options](#opt
173
- `[prometheus:exporter]` defines settings for Prometheus exporter API queries (e.g.:
174
`http://NODE:19999/api/v1/allmetrics?format=prometheus&help=yes&source=as-collected`).
175
- `[<type>:<name>]` keeps settings for a particular exporting connector instance, where:
176
- - `type` selects the exporting connector type: graphite | opentsdb:telnet | opentsdb:http | opentsdb:https |
177
- prometheus_remote_write | json | kinesis | pubsub | mongodb
176
+ - `type` selects the exporting connector type: graphite | opentsdb:telnet | opentsdb:http |
177
+ prometheus_remote_write | json | kinesis | pubsub | mongodb. For graphite, opentsdb,
178
+ json, and prometheus_remote_write connectors you can also use `:http` or `:https` modifiers
179
+ (e.g.: `opentsdb:https`).
180
- `name` can be arbitrary instance name you chose.
181
182
### Options
@@ -270,6 +272,13 @@ Configure individual connectors and override any global settings with the follow
272
> You can check how the host tags were parsed using the /api/v1/info API call. But, keep in mind that backends subsystem
273
> is deprecated and will be deleted soon. Please move your existing tags to the `[host labels]` section.
274
275
+## HTTPS
276
+
277
+Netdata can send metrics to external databases using the TLS/SSL protocol. Unfortunately, some of
278
+them does not support encrypted connections, so you will have to configure a reverse proxy to enable
279
+HTTPS communication between Netdata and an external database. You can set up a reverse proxy with
280
+[Nginx](/docs/Running-behind-nginx.md).
281
+
282
## Exporting engine monitoring
283
284
Netdata creates five charts in the dashboard, under the **Netdata Monitoring** section, to help you monitor the health
exporting/aws_kinesis/aws_kinesis.c
+1
-1
@@ -48,7 +48,7 @@ int init_aws_kinesis_instance(struct instance *instance)
48
instance->end_host_formatting = flush_host_labels;
49
instance->end_batch_formatting = NULL;
50
51
- instance->send_header = NULL;
51
+ instance->prepare_header = NULL;
52
instance->check_response = NULL;
53
54
instance->buffer = (void *)buffer_create(0);
exporting/clean_connectors.c
+43
@@ -35,3 +35,46 @@ void clean_instance(struct instance *instance)
35
uv_cond_destroy(&instance->cond_var);
36
// uv_mutex_destroy(&instance->mutex);
37
}
38
+
39
+/**
40
+ * Clean up a simple connector instance on Netdata exit
41
+ *
42
+ * @param instance an instance data structure.
43
+ */
44
+void simple_connector_cleanup(struct instance *instance)
45
+{
46
+ info("EXPORTING: cleaning up instance %s ...", instance->config.name);
47
+
48
+ struct simple_connector_data *simple_connector_data =
49
+ (struct simple_connector_data *)instance->connector_specific_data;
50
+
51
+ buffer_free(instance->buffer);
52
+ buffer_free(simple_connector_data->buffer);
53
+ buffer_free(simple_connector_data->header);
54
+
55
+ struct simple_connector_buffer *next_buffer = simple_connector_data->first_buffer;
56
+ for (int i = 0; i < instance->config.buffer_on_failures; i++) {
57
+ struct simple_connector_buffer *current_buffer = next_buffer;
58
+ next_buffer = next_buffer->next;
59
+
60
+ buffer_free(current_buffer->header);
61
+ buffer_free(current_buffer->buffer);
62
+ freez(current_buffer);
63
+ }
64
+
65
+#ifdef ENABLE_HTTPS
66
+ if (simple_connector_data->conn)
67
+ SSL_free(simple_connector_data->conn);
68
+#endif
69
+
70
+ freez(simple_connector_data);
71
+
72
+ struct simple_connector_config *simple_connector_config =
73
+ (struct simple_connector_config *)instance->config.connector_specific_config;
74
+ freez(simple_connector_config);
75
+
76
+ info("EXPORTING: instance %s exited", instance->config.name);
77
+ instance->exited = 1;
78
+
79
+ return;
80
+}
exporting/exporting_engine.h
+44
-7
@@ -46,10 +46,12 @@ typedef enum exporting_options {
46
typedef enum exporting_connector_types {
47
EXPORTING_CONNECTOR_TYPE_UNKNOWN, // Invalid type
48
EXPORTING_CONNECTOR_TYPE_GRAPHITE, // Send plain text to Graphite
49
- EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_TELNET, // Send data to OpenTSDB using telnet API
50
- EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP, // Send data to OpenTSDB using HTTP API
51
- EXPORTING_CONNECTOR_TYPE_JSON, // Stores the data using JSON.
52
- EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE, // The user selected to use Prometheus backend
49
+ EXPORTING_CONNECTOR_TYPE_GRAPHITE_HTTP, // Send data to Graphite using HTTP API
50
+ EXPORTING_CONNECTOR_TYPE_JSON, // Send data in JSON format
51
+ EXPORTING_CONNECTOR_TYPE_JSON_HTTP, // Send data in JSON format using HTTP API
52
+ EXPORTING_CONNECTOR_TYPE_OPENTSDB, // Send data to OpenTSDB using telnet API
53
+ EXPORTING_CONNECTOR_TYPE_OPENTSDB_HTTP, // Send data to OpenTSDB using HTTP API
54
+ EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE, // User selected to use Prometheus backend
55
EXPORTING_CONNECTOR_TYPE_KINESIS, // Send message to AWS Kinesis
56
EXPORTING_CONNECTOR_TYPE_PUBSUB, // Send message to Google Cloud Pub/Sub
57
EXPORTING_CONNECTOR_TYPE_MONGODB, // Send data to MongoDB collection
@@ -81,6 +83,38 @@ struct simple_connector_config {
83
int default_port;
84
};
85
86
+struct simple_connector_buffer {
87
+ BUFFER *header;
88
+ BUFFER *buffer;
89
+
90
+ size_t buffered_metrics;
91
+ size_t buffered_bytes;
92
+
93
+ int used;
94
+
95
+ struct simple_connector_buffer *next;
96
+};
97
+
98
+struct simple_connector_data {
99
+ void *connector_specific_data;
100
+
101
+ size_t total_buffered_metrics;
102
+
103
+ BUFFER *header;
104
+ BUFFER *buffer;
105
+ size_t buffered_metrics;
106
+ size_t buffered_bytes;
107
+
108
+ struct simple_connector_buffer *previous_buffer;
109
+ struct simple_connector_buffer *first_buffer;
110
+ struct simple_connector_buffer *last_buffer;
111
+
112
+#ifdef ENABLE_HTTPS
113
+ SSL *conn; //SSL connection
114
+ int flags; //The flags for SSL connection
115
+#endif
116
+};
117
+
118
struct prometheus_remote_write_specific_config {
119
char *remote_write_path;
120
};
@@ -175,7 +209,7 @@ struct instance {
209
int (*end_host_formatting)(struct instance *instance, RRDHOST *host);
210
int (*end_batch_formatting)(struct instance *instance);
211
178
- int (*send_header)(int *sock, struct instance *instance);
212
+ void (*prepare_header)(struct instance *instance);
213
int (*check_response)(BUFFER *buffer, struct instance *instance);
214
215
void *connector_specific_data;
@@ -210,6 +244,7 @@ struct engine *read_exporting_config();
244
EXPORTING_CONNECTOR_TYPE exporting_select_type(const char *type);
245
246
int init_connectors(struct engine *engine);
247
+void simple_connector_init(struct instance *instance);
248
249
int mark_scheduled_instances(struct engine *engine);
250
void prepare_buffers(struct engine *engine);
@@ -232,11 +267,12 @@ void end_chart_formatting(struct engine *engine, RRDSET *st);
267
void end_host_formatting(struct engine *engine, RRDHOST *host);
268
void end_batch_formatting(struct engine *engine);
269
int flush_host_labels(struct instance *instance, RRDHOST *host);
235
-int simple_connector_update_buffered_bytes(struct instance *instance);
270
+int simple_connector_end_batch(struct instance *instance);
271
272
int exporting_discard_response(BUFFER *buffer, struct instance *instance);
273
void simple_connector_receive_response(int *sock, struct instance *instance);
239
-void simple_connector_send_buffer(int *sock, int *failures, struct instance *instance);
274
+void simple_connector_send_buffer(
275
+ int *sock, int *failures, struct instance *instance, BUFFER *header, BUFFER *buffer, size_t buffered_metrics);
276
void simple_connector_worker(void *instance_p);
277
278
void create_main_rusage_chart(RRDSET **st_rusage, RRDDIM **rd_user, RRDDIM **rd_system);
@@ -244,6 +280,7 @@ void send_main_rusage(RRDSET *st_rusage, RRDDIM *rd_user, RRDDIM *rd_system);
280
void send_internal_metrics(struct instance *instance);
281
282
extern void clean_instance(struct instance *ptr);
283
+void simple_connector_cleanup(struct instance *instance);
284
285
static inline void disable_instance(struct instance *instance)
286
{
exporting/graphite/README.md
+4
@@ -3,6 +3,7 @@ title: "Export metrics to Graphite providers"
3
sidebar_label: Graphite
4
description: "Archive your Agent's metrics to a any Graphite database provider for long-term storage, further analysis, or correlation with data from other sources."
5
custom_edit_url: https://github.com/netdata/netdata/edit/master/exporting/graphite/README.md
6
+sidebar_label: Graphite
7
-->
8
9
# Export metrics to Graphite providers
@@ -21,6 +22,9 @@ directory and set the following options:
22
destination = localhost:2003
23
```
24
25
+Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `graphite:http:my_graphite_instance`,
26
+`graphite:https:my_graphite_instance`.
27
+
28
The Graphite connector is further configurable using additional settings. See the [exporting reference
29
doc](/exporting/README.md#options) for details.
30
exporting/graphite/graphite.c
+43
-2
@@ -16,6 +16,17 @@ int init_graphite_instance(struct instance *instance)
16
instance->config.connector_specific_config = (void *)connector_specific_config;
17
connector_specific_config->default_port = 2003;
18
19
+ struct simple_connector_data *connector_specific_data = callocz(1, sizeof(struct simple_connector_data));
20
+ instance->connector_specific_data = connector_specific_data;
21
+
22
+#ifdef ENABLE_HTTPS
23
+ connector_specific_data->flags = NETDATA_SSL_START;
24
+ connector_specific_data->conn = NULL;
25
+ if (instance->config.options & EXPORTING_OPTION_USE_TLS) {
26
+ security_start_ssl(NETDATA_SSL_CONTEXT_EXPORTING);
27
+ }
28
+#endif
29
+
30
instance->start_batch_formatting = NULL;
31
instance->start_host_formatting = format_host_labels_graphite_plaintext;
32
instance->start_chart_formatting = NULL;
@@ -27,9 +38,13 @@ int init_graphite_instance(struct instance *instance)
38
39
instance->end_chart_formatting = NULL;
40
instance->end_host_formatting = flush_host_labels;
30
- instance->end_batch_formatting = simple_connector_update_buffered_bytes;
41
+ instance->end_batch_formatting = simple_connector_end_batch;
42
+
43
+ if (instance->config.type == EXPORTING_CONNECTOR_TYPE_GRAPHITE_HTTP)
44
+ instance->prepare_header = graphite_http_prepare_header;
45
+ else
46
+ instance->prepare_header = NULL;
47
32
- instance->send_header = NULL;
48
instance->check_response = exporting_discard_response;
49
50
instance->buffer = (void *)buffer_create(0);
@@ -37,6 +52,9 @@ int init_graphite_instance(struct instance *instance)
52
error("EXPORTING: cannot create buffer for graphite exporting connector instance %s", instance->config.name);
53
return 1;
54
}
55
+
56
+ simple_connector_init(instance);
57
+
58
if (uv_mutex_init(&instance->mutex))
59
return 1;
60
if (uv_cond_init(&instance->cond_var))
@@ -187,3 +205,26 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
205
206
return 0;
207
}
208
+
209
+/**
210
+ * Ppepare HTTP header
211
+ *
212
+ * @param instance an instance data structure.
213
+ * @return Returns 0 on success, 1 on failure.
214
+ */
215
+void graphite_http_prepare_header(struct instance *instance)
216
+{
217
+ struct simple_connector_data *simple_connector_data = instance->connector_specific_data;
218
+
219
+ buffer_sprintf(
220
+ simple_connector_data->last_buffer->header,
221
+ "POST /api/put HTTP/1.1\r\n"
222
+ "Host: %s\r\n"
223
+ "Content-Type: application/graphite\r\n"
224
+ "Content-Length: %lu\r\n"
225
+ "\r\n",
226
+ instance->config.destination,
227
+ buffer_strlen(simple_connector_data->last_buffer->buffer));
228
+
229
+ return;
230
+}
exporting/graphite/graphite.h
+2
@@ -13,4 +13,6 @@ int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *ho
13
int format_dimension_collected_graphite_plaintext(struct instance *instance, RRDDIM *rd);
14
int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM *rd);
15
16
+void graphite_http_prepare_header(struct instance *instance);
17
+
18
#endif //NETDATA_EXPORTING_GRAPHITE_H
exporting/init_connectors.c
+43
-2
@@ -40,15 +40,23 @@ int init_connectors(struct engine *engine)
40
if (init_graphite_instance(instance) != 0)
41
return 1;
42
break;
43
+ case EXPORTING_CONNECTOR_TYPE_GRAPHITE_HTTP:
44
+ if (init_graphite_instance(instance) != 0)
45
+ return 1;
46
+ break;
47
case EXPORTING_CONNECTOR_TYPE_JSON:
48
if (init_json_instance(instance) != 0)
49
return 1;
50
break;
47
- case EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_TELNET:
51
+ case EXPORTING_CONNECTOR_TYPE_JSON_HTTP:
52
+ if (init_json_http_instance(instance) != 0)
53
+ return 1;
54
+ break;
55
+ case EXPORTING_CONNECTOR_TYPE_OPENTSDB:
56
if (init_opentsdb_telnet_instance(instance) != 0)
57
return 1;
58
break;
51
- case EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP:
59
+ case EXPORTING_CONNECTOR_TYPE_OPENTSDB_HTTP:
60
if (init_opentsdb_http_instance(instance) != 0)
61
return 1;
62
break;
@@ -96,3 +104,36 @@ int init_connectors(struct engine *engine)
104
105
return 0;
106
}
107
+
108
+/**
109
+ * Initialize a ring buffer for a simple connector
110
+ *
111
+ * @param instance an instance data structure.
112
+ */
113
+void simple_connector_init(struct instance *instance)
114
+{
115
+ struct simple_connector_data *connector_specific_data =
116
+ (struct simple_connector_data *)instance->connector_specific_data;
117
+
118
+ // create a ring buffer
119
+ struct simple_connector_buffer *first_buffer = NULL;
120
+
121
+ if (instance->config.buffer_on_failures < 1)
122
+ instance->config.buffer_on_failures = 1;
123
+
124
+ for (int i = 0; i < instance->config.buffer_on_failures; i++) {
125
+ struct simple_connector_buffer *current_buffer = callocz(1, sizeof(struct simple_connector_buffer));
126
+
127
+ if (!connector_specific_data->first_buffer)
128
+ first_buffer = current_buffer;
129
+ else
130
+ current_buffer->next = connector_specific_data->first_buffer;
131
+
132
+ connector_specific_data->first_buffer = current_buffer;
133
+ }
134
+
135
+ first_buffer->next = connector_specific_data->first_buffer;
136
+ connector_specific_data->last_buffer = connector_specific_data->first_buffer;
137
+
138
+ return;
139
+}
exporting/json/README.md
+4
@@ -3,6 +3,7 @@ title: "Export metrics to JSON document databases"
3
sidebar_label: JSON
4
description: "Archive your Agent's metrics to a JSON document database for long-term storage, further analysis, or correlation with data from other sources."
5
custom_edit_url: https://github.com/netdata/netdata/edit/master/exporting/json/README.md
6
+sidebar_label: JSON Document Databases
7
-->
8
9
# Export metrics to JSON document databases
@@ -21,6 +22,9 @@ directory and set the following options:
22
destination = localhost:5448
23
```
24
25
+Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `json:http:my_json_instance`,
26
+`json:https:my_json_instance`.
27
+
28
The JSON connector is further configurable using additional settings. See the [exporting reference
29
doc](/exporting/README.md#options) for details.
30
exporting/json/json.c
+134
-4
@@ -16,6 +16,9 @@ int init_json_instance(struct instance *instance)
16
instance->config.connector_specific_config = (void *)connector_specific_config;
17
connector_specific_config->default_port = 5448;
18
19
+ struct simple_connector_data *connector_specific_data = callocz(1, sizeof(struct simple_connector_data));
20
+ instance->connector_specific_data = connector_specific_data;
21
+
22
instance->start_batch_formatting = NULL;
23
instance->start_host_formatting = format_host_labels_json_plaintext;
24
instance->start_chart_formatting = NULL;
@@ -27,9 +30,10 @@ int init_json_instance(struct instance *instance)
30
31
instance->end_chart_formatting = NULL;
32
instance->end_host_formatting = flush_host_labels;
30
- instance->end_batch_formatting = simple_connector_update_buffered_bytes;
33
+ instance->end_batch_formatting = simple_connector_end_batch;
34
+
35
+ instance->prepare_header = NULL;
36
32
- instance->send_header = NULL;
37
instance->check_response = exporting_discard_response;
38
39
instance->buffer = (void *)buffer_create(0);
@@ -37,6 +41,63 @@ int init_json_instance(struct instance *instance)
41
error("EXPORTING: cannot create buffer for json exporting connector instance %s", instance->config.name);
42
return 1;
43
}
44
+
45
+ simple_connector_init(instance);
46
+
47
+ if (uv_mutex_init(&instance->mutex))
48
+ return 1;
49
+ if (uv_cond_init(&instance->cond_var))
50
+ return 1;
51
+
52
+ return 0;
53
+}
54
+
55
+/**
56
+ * Initialize JSON connector instance for HTTP protocol
57
+ *
58
+ * @param instance an instance data structure.
59
+ * @return Returns 0 on success, 1 on failure.
60
+ */
61
+int init_json_http_instance(struct instance *instance)
62
+{
63
+ instance->worker = simple_connector_worker;
64
+
65
+ struct simple_connector_config *connector_specific_config = callocz(1, sizeof(struct simple_connector_config));
66
+ instance->config.connector_specific_config = (void *)connector_specific_config;
67
+ connector_specific_config->default_port = 5448;
68
+
69
+ struct simple_connector_data *connector_specific_data = callocz(1, sizeof(struct simple_connector_data));
70
+ instance->connector_specific_data = connector_specific_data;
71
+
72
+#ifdef ENABLE_HTTPS
73
+ connector_specific_data->flags = NETDATA_SSL_START;
74
+ connector_specific_data->conn = NULL;
75
+ if (instance->config.options & EXPORTING_OPTION_USE_TLS) {
76
+ security_start_ssl(NETDATA_SSL_CONTEXT_EXPORTING);
77
+ }
78
+#endif
79
+
80
+ instance->start_batch_formatting = open_batch_json_http;
81
+ instance->start_host_formatting = format_host_labels_json_plaintext;
82
+ instance->start_chart_formatting = NULL;
83
+
84
+ if (EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED)
85
+ instance->metric_formatting = format_dimension_collected_json_plaintext;
86
+ else
87
+ instance->metric_formatting = format_dimension_stored_json_plaintext;
88
+
89
+ instance->end_chart_formatting = NULL;
90
+ instance->end_host_formatting = flush_host_labels;
91
+ instance->end_batch_formatting = close_batch_json_http;
92
+
93
+ instance->prepare_header = json_http_prepare_header;
94
+
95
+ instance->check_response = exporting_discard_response;
96
+
97
+ instance->buffer = (void *)buffer_create(0);
98
+
99
+ simple_connector_init(instance);
100
+
101
if (uv_mutex_init(&instance->mutex))
102
return 1;
103
if (uv_cond_init(&instance->cond_var))
@@ -111,6 +172,11 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
172
}
173
}
174
175
+ if (instance->config.type == EXPORTING_CONNECTOR_TYPE_JSON_HTTP) {
176
+ if (buffer_strlen((BUFFER *)instance->buffer) > 2)
177
+ buffer_strcat(instance->buffer, ",\n");
178
+ }
179
+
180
buffer_sprintf(
181
instance->buffer,
182
@@ -131,7 +197,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
197
"\"name\":\"%s\","
198
"\"value\":" COLLECTED_NUMBER_FORMAT ","
199
134
- "\"timestamp\":%llu}\n",
200
+ "\"timestamp\":%llu}",
201
202
instance->config.prefix,
203
(host == localhost) ? engine->config.hostname : host->hostname,
@@ -153,6 +219,10 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
219
220
(unsigned long long)rd->last_collected_time.tv_sec);
221
222
+ if (instance->config.type != EXPORTING_CONNECTOR_TYPE_JSON_HTTP) {
223
+ buffer_strcat(instance->buffer, "\n");
224
+ }
225
+
226
return 0;
227
}
228
@@ -189,6 +259,11 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
259
}
260
}
261
262
+ if (instance->config.type == EXPORTING_CONNECTOR_TYPE_JSON_HTTP) {
263
+ if (buffer_strlen((BUFFER *)instance->buffer) > 2)
264
+ buffer_strcat(instance->buffer, ",\n");
265
+ }
266
+
267
buffer_sprintf(
268
instance->buffer,
269
"{"
@@ -208,7 +283,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
283
"\"name\":\"%s\","
284
"\"value\":" CALCULATED_NUMBER_FORMAT ","
285
211
- "\"timestamp\": %llu}\n",
286
+ "\"timestamp\": %llu}",
287
288
instance->config.prefix,
289
(host == localhost) ? engine->config.hostname : host->hostname,
@@ -230,5 +305,60 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
305
306
(unsigned long long)last_t);
307
308
+ if (instance->config.type != EXPORTING_CONNECTOR_TYPE_JSON_HTTP) {
309
+ buffer_strcat(instance->buffer, "\n");
310
+ }
311
+
312
return 0;
313
}
314
+
315
+/**
316
+ * Open a JSON list for a bach
317
+ *
318
+ * @param instance an instance data structure.
319
+ * @return Always returns 0.
320
+ */
321
+int open_batch_json_http(struct instance *instance)
322
+{
323
+ buffer_strcat(instance->buffer, "[\n");
324
+
325
+ return 0;
326
+}
327
+
328
+/**
329
+ * Close a JSON list for a bach and update buffered bytes counter
330
+ *
331
+ * @param instance an instance data structure.
332
+ * @return Always returns 0.
333
+ */
334
+int close_batch_json_http(struct instance *instance)
335
+{
336
+ buffer_strcat(instance->buffer, "\n]\n");
337
+
338
+ simple_connector_end_batch(instance);
339
+
340
+ return 0;
341
+}
342
+
343
+/**
344
+ * Prepare HTTP header
345
+ *
346
+ * @param instance an instance data structure.
347
+ * @return Returns 0 on success, 1 on failure.
348
+ */
349
+void json_http_prepare_header(struct instance *instance)
350
+{
351
+ struct simple_connector_data *simple_connector_data = instance->connector_specific_data;
352
+
353
+ buffer_sprintf(
354
+ simple_connector_data->last_buffer->header,
355
+ "POST /api/put HTTP/1.1\r\n"
356
+ "Host: %s\r\n"
357
+ "Content-Type: application/json\r\n"
358
+ "Content-Length: %lu\r\n"
359
+ "\r\n",
360
+ instance->config.destination,
361
+ buffer_strlen(simple_connector_data->last_buffer->buffer));
362
+
363
+ return;
364
+}
exporting/json/json.h
+6
@@ -6,10 +6,16 @@
6
#include "exporting/exporting_engine.h"
7
8
int init_json_instance(struct instance *instance);
9
+int init_json_http_instance(struct instance *instance);
10
11
int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host);
12
13
int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM *rd);
14
int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd);
15
16
+int open_batch_json_http(struct instance *instance);
17
+int close_batch_json_http(struct instance *instance);
18
+
19
+void json_http_prepare_header(struct instance *instance);
20
+
21
#endif //NETDATA_EXPORTING_JSON_H
exporting/mongodb/mongodb.c
+7
-4
@@ -102,7 +102,7 @@ int init_mongodb_instance(struct instance *instance)
102
instance->end_host_formatting = flush_host_labels;
103
instance->end_batch_formatting = format_batch_mongodb;
104
105
- instance->send_header = NULL;
105
+ instance->prepare_header = NULL;
106
instance->check_response = NULL;
107
108
instance->buffer = (void *)buffer_create(0);
@@ -284,9 +284,12 @@ void mongodb_connector_worker(void *instance_p)
284
struct stats *stats = &instance->stats;
285
286
uv_mutex_lock(&instance->mutex);
287
- while (!instance->data_is_ready)
288
- uv_cond_wait(&instance->cond_var, &instance->mutex);
289
- instance->data_is_ready = 0;
287
+ if (!connector_specific_data->first_buffer->insert ||
288
+ !connector_specific_data->first_buffer->documents_inserted) {
289
+ while (!instance->data_is_ready)
290
+ uv_cond_wait(&instance->cond_var, &instance->mutex);
291
+ instance->data_is_ready = 0;
292
+ }
293
294
if (unlikely(instance->engine->exit)) {
295
uv_mutex_unlock(&instance->mutex);
exporting/mongodb/mongodb.h
-1
@@ -21,7 +21,6 @@ struct mongodb_specific_data {
21
22
size_t total_documents_inserted;
23
24
- bson_t **current_insert;
24
struct bson_buffer *first_buffer;
25
struct bson_buffer *last_buffer;
26
};
exporting/opentsdb/README.md
+16
-26
@@ -1,40 +1,30 @@
1
<!--
2
-title: "Export metrics to OpenTSDB with HTTP"
3
-description: "Archive your Agent's metrics to a OpenTSDB database for long-term storage and further analysis."
2
+title: "Export metrics to OpenTSDB"
3
+description: "Archive your Agent's metrics to an OpenTSDB database for long-term storage and further analysis."
4
custom_edit_url: https://github.com/netdata/netdata/edit/master/exporting/opentsdb/README.md
5
-sidebar_label: OpenTSDB with HTTP
5
+sidebar_label: OpenTSDB
6
-->
7
8
-# Export metrics to OpenTSDB with HTTP
8
+# Export metrics to OpenTSDB
9
10
-Netdata can easily communicate with OpenTSDB using HTTP API. To enable this channel, run `./edit-config exporting.conf`
11
-in the Netdata configuration directory and set the following options:
10
+You can use the OpenTSDB connector for the [exporting engine](/exporting/README.md) to archive your agent's metrics to OpenTSDB
11
+databases for long-term storage, further analysis, or correlation with data from other sources.
12
13
-```conf
14
-[opentsdb:http:my_instance]
15
- enabled = yes
16
- destination = localhost:4242
17
-```
18
-
19
-In this example, OpenTSDB is running with its default port, which is `4242`. If you run OpenTSDB on a different port,
20
-change the `destination = localhost:4242` line accordingly.
13
+## Configuration
14
22
-## HTTPS
23
-
24
-As of [v1.16.0](https://github.com/netdata/netdata/releases/tag/v1.16.0), Netdata can send metrics to OpenTSDB using
25
-TLS/SSL. Unfortunately, OpenTDSB does not support encrypted connections, so you will have to configure a reverse proxy
26
-to enable HTTPS communication between Netdata and OpenTSBD. You can set up a reverse proxy with
27
-[Nginx](/docs/Running-behind-nginx.md).
28
-
29
-After your proxy is configured, make the following changes to `exporting.conf`:
15
+To enable data exporting to an OpenTSDB database, run `./edit-config exporting.conf` in the Netdata configuration
16
+directory and set the following options:
17
18
```conf
32
-[opentsdb:https:my_instance]
19
+[opentsdb:my_opentsdb_instance]
20
enabled = yes
34
- destination = localhost:8082
21
+ destination = localhost:4242
22
```
23
37
-In this example, we used the port `8082` for our reverse proxy. If your reverse proxy listens on a different port,
38
-change the `destination = localhost:8082` line accordingly.
24
+Add `:http` or `:https` modifiers to the connector type if you need to use other than a plaintext protocol. For example: `opentsdb:http:my_opentsdb_instance`,
25
+`opentsdb:https:my_opentsdb_instance`.
26
+
27
+The OpenTSDB connector is further configurable using additional settings. See the [exporting reference
28
+doc](/exporting/README.md#options) for details.
29
30
[](<>)
exporting/opentsdb/opentsdb.c
+61
-49
@@ -1,6 +1,7 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
#include "opentsdb.h"
4
+#include "../json/json.h"
5
6
/**
7
* Initialize OpenTSDB telnet connector instance
@@ -16,6 +17,17 @@ int init_opentsdb_telnet_instance(struct instance *instance)
17
instance->config.connector_specific_config = (void *)connector_specific_config;
18
connector_specific_config->default_port = 4242;
19
20
+ struct simple_connector_data *connector_specific_data = callocz(1, sizeof(struct simple_connector_data));
21
+ instance->connector_specific_data = connector_specific_data;
22
+
23
+#ifdef ENABLE_HTTPS
24
+ connector_specific_data->flags = NETDATA_SSL_START;
25
+ connector_specific_data->conn = NULL;
26
+ if (instance->config.options & EXPORTING_OPTION_USE_TLS) {
27
+ security_start_ssl(NETDATA_SSL_CONTEXT_EXPORTING);
28
+ }
29
+#endif
30
+
31
instance->start_batch_formatting = NULL;
32
instance->start_host_formatting = format_host_labels_opentsdb_telnet;
33
instance->start_chart_formatting = NULL;
@@ -27,9 +39,9 @@ int init_opentsdb_telnet_instance(struct instance *instance)
39
40
instance->end_chart_formatting = NULL;
41
instance->end_host_formatting = flush_host_labels;
30
- instance->end_batch_formatting = simple_connector_update_buffered_bytes;
42
+ instance->end_batch_formatting = simple_connector_end_batch;
43
32
- instance->send_header = NULL;
44
+ instance->prepare_header = NULL;
45
instance->check_response = exporting_discard_response;
46
47
instance->buffer = (void *)buffer_create(0);
@@ -37,6 +49,9 @@ int init_opentsdb_telnet_instance(struct instance *instance)
49
error("EXPORTING: cannot create buffer for opentsdb telnet exporting connector instance %s", instance->config.name);
50
return 1;
51
}
52
+
53
+ simple_connector_init(instance);
54
+
55
if (uv_mutex_init(&instance->mutex))
56
return 1;
57
if (uv_cond_init(&instance->cond_var))
@@ -59,17 +74,17 @@ int init_opentsdb_http_instance(struct instance *instance)
74
instance->config.connector_specific_config = (void *)connector_specific_config;
75
connector_specific_config->default_port = 4242;
76
77
+ struct simple_connector_data *connector_specific_data = callocz(1, sizeof(struct simple_connector_data));
78
#ifdef ENABLE_HTTPS
63
- struct opentsdb_specific_data *connector_specific_data = callocz(1, sizeof(struct opentsdb_specific_data));
79
connector_specific_data->flags = NETDATA_SSL_START;
80
connector_specific_data->conn = NULL;
81
if (instance->config.options & EXPORTING_OPTION_USE_TLS) {
67
- security_start_ssl(NETDATA_SSL_CONTEXT_OPENTSDB);
82
+ security_start_ssl(NETDATA_SSL_CONTEXT_EXPORTING);
83
}
69
- instance->connector_specific_data = connector_specific_data;
84
#endif
85
+ instance->connector_specific_data = connector_specific_data;
86
72
- instance->start_batch_formatting = NULL;
87
+ instance->start_batch_formatting = open_batch_json_http;
88
instance->start_host_formatting = format_host_labels_opentsdb_http;
89
instance->start_chart_formatting = NULL;
90
@@ -80,9 +95,9 @@ int init_opentsdb_http_instance(struct instance *instance)
95
96
instance->end_chart_formatting = NULL;
97
instance->end_host_formatting = flush_host_labels;
83
- instance->end_batch_formatting = simple_connector_update_buffered_bytes;
98
+ instance->end_batch_formatting = close_batch_json_http;
99
85
- instance->send_header = NULL;
100
+ instance->prepare_header = opentsdb_http_prepare_header;
101
instance->check_response = exporting_discard_response;
102
103
instance->buffer = (void *)buffer_create(0);
@@ -90,6 +105,9 @@ int init_opentsdb_http_instance(struct instance *instance)
105
error("EXPORTING: cannot create buffer for opentsdb HTTP exporting connector instance %s", instance->config.name);
106
return 1;
107
}
108
+
109
+ simple_connector_init(instance);
110
+
111
if (uv_mutex_init(&instance->mutex))
112
return 1;
113
if (uv_cond_init(&instance->cond_var))
@@ -240,26 +258,26 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
258
}
259
260
/**
243
- * Prepare an HTTP message for OpenTSDB HTTP connector
261
+ * Ppepare HTTP header
262
*
245
- * @param buffer a buffer to write the message to.
246
- * @param message the body of the message.
247
- * @param hostname the name of the host that sends the message.
248
- * @param length the length of the message body.
263
+ * @param instance an instance data structure.
264
+ * @return Returns 0 on success, 1 on failure.
265
*/
250
-static inline void opentsdb_build_message(BUFFER *buffer, char *message, const char *hostname, int length)
266
+void opentsdb_http_prepare_header(struct instance *instance)
267
{
268
+ struct simple_connector_data *simple_connector_data = instance->connector_specific_data;
269
+
270
buffer_sprintf(
253
- buffer,
271
+ simple_connector_data->last_buffer->header,
272
"POST /api/put HTTP/1.1\r\n"
273
"Host: %s\r\n"
274
"Content-Type: application/json\r\n"
257
- "Content-Length: %d\r\n"
258
- "\r\n"
259
- "%s",
260
- hostname,
261
- length,
262
- message);
275
+ "Content-Length: %lu\r\n"
276
+ "\r\n",
277
+ instance->config.destination,
278
+ buffer_strlen(simple_connector_data->last_buffer->buffer));
279
+
280
+ return;
281
}
282
283
/**
@@ -324,17 +342,18 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
342
(instance->config.options & EXPORTING_OPTION_SEND_NAMES && rd->name) ? rd->name : rd->id,
343
RRD_ID_LENGTH_MAX);
344
327
- char message[1024];
328
- int length = snprintfz(
329
- message,
330
- sizeof(message),
345
+ if (buffer_strlen((BUFFER *)instance->buffer) > 2)
346
+ buffer_strcat(instance->buffer, ",\n");
347
+
348
+ buffer_sprintf(
349
+ instance->buffer,
350
"{"
332
- " \"metric\": \"%s.%s.%s\","
333
- " \"timestamp\": %llu,"
334
- " \"value\": " COLLECTED_NUMBER_FORMAT ","
335
- " \"tags\": {"
336
- " \"host\": \"%s%s%s\"%s"
337
- " }"
351
+ "\"metric\":\"%s.%s.%s\","
352
+ "\"timestamp\":%llu,"
353
+ "\"value\":"COLLECTED_NUMBER_FORMAT","
354
+ "\"tags\":{"
355
+ "\"host\":\"%s%s%s\"%s"
356
+ "}"
357
"}",
358
instance->config.prefix,
359
chart_name,
@@ -346,10 +365,6 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
365
(host->tags) ? host->tags : "",
366
instance->labels ? buffer_tostring(instance->labels) : "");
367
349
- if (length > 0) {
350
- opentsdb_build_message(instance->buffer, message, engine->config.hostname, length);
351
- }
352
-
368
return 0;
369
}
370
@@ -384,17 +399,18 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
399
if(isnan(value))
400
return 0;
401
387
- char message[1024];
388
- int length = snprintfz(
389
- message,
390
- sizeof(message),
402
+ if (buffer_strlen((BUFFER *)instance->buffer) > 2)
403
+ buffer_strcat(instance->buffer, ",\n");
404
+
405
+ buffer_sprintf(
406
+ instance->buffer,
407
"{"
392
- " \"metric\": \"%s.%s.%s\","
393
- " \"timestamp\": %llu,"
394
- " \"value\": " CALCULATED_NUMBER_FORMAT ","
395
- " \"tags\": {"
396
- " \"host\": \"%s%s%s\"%s"
397
- " }"
408
+ "\"metric\":\"%s.%s.%s\","
409
+ "\"timestamp\":%llu,"
410
+ "\"value\":"CALCULATED_NUMBER_FORMAT","
411
+ "\"tags\":{"
412
+ "\"host\":\"%s%s%s\"%s"
413
+ "}"
414
"}",
415
instance->config.prefix,
416
chart_name,
@@ -406,9 +422,5 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
422
(host->tags) ? host->tags : "",
423
instance->labels ? buffer_tostring(instance->labels) : "");
424
409
- if (length > 0) {
410
- opentsdb_build_message(instance->buffer, message, engine->config.hostname, length);
411
- }
412
-
425
return 0;
426
}
exporting/opentsdb/opentsdb.h
+4
-6
@@ -18,11 +18,9 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
18
int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *rd);
19
int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd);
20
21
-#ifdef ENABLE_HTTPS
22
-struct opentsdb_specific_data {
23
- SSL *conn; //SSL connection
24
- int flags; //The flags for SSL connection
25
-};
26
-#endif
21
+int open_batch_opentsdb_http(struct instance *instance);
22
+int close_batch_opentsdb_http(struct instance *instance);
23
+
24
+void opentsdb_http_prepare_header(struct instance *instance);
25
26
#endif //NETDATA_EXPORTING_OPENTSDB_H
exporting/process_data.c
+54
-4
@@ -363,14 +363,64 @@ int flush_host_labels(struct instance *instance, RRDHOST *host)
363
}
364
365
/**
366
- * Update stats for buffered bytes
366
+ * End a batch for a simple connector
367
*
368
* @param instance an instance data structure.
369
- * @return Always returns 0.
369
+ * @return Returns 0 on success, 1 on failure.
370
*/
371
-int simple_connector_update_buffered_bytes(struct instance *instance)
371
+int simple_connector_end_batch(struct instance *instance)
372
{
373
- instance->stats.buffered_bytes = (collected_number)buffer_strlen((BUFFER *)(instance->buffer));
373
+ struct simple_connector_data *simple_connector_data =
374
+ (struct simple_connector_data *)instance->connector_specific_data;
375
+ struct stats *stats = &instance->stats;
376
+
377
+ BUFFER *instance_buffer = (BUFFER *)instance->buffer;
378
+ struct simple_connector_buffer *last_buffer = simple_connector_data->last_buffer;
379
+
380
+ if (!last_buffer->buffer) {
381
+ last_buffer->buffer = buffer_create(0);
382
+ }
383
+
384
+ if (last_buffer->used) {
385
+ // ring buffer is full, reuse the oldest element
386
+ simple_connector_data->first_buffer = simple_connector_data->first_buffer->next;
387
+
388
+ stats->data_lost_events++;
389
+ stats->lost_metrics += last_buffer->buffered_metrics;
390
+ stats->lost_bytes += last_buffer->buffered_bytes;
391
+ }
392
+
393
+ // swap buffers
394
+ BUFFER *tmp_buffer = last_buffer->buffer;
395
+ last_buffer->buffer = instance_buffer;
396
+ instance->buffer = instance_buffer = tmp_buffer;
397
+
398
+ buffer_flush(instance_buffer);
399
+
400
+ if (last_buffer->header)
401
+ buffer_flush(last_buffer->header);
402
+ else
403
+ last_buffer->header = buffer_create(0);
404
+
405
+ if (instance->prepare_header)
406
+ instance->prepare_header(instance);
407
+
408
+ // The stats->buffered_metrics is used in the simple connector batch formatting as a variable for the number
409
+ // of metrics, added in the current iteration, so we are clearing it here. We will use the
410
+ // simple_connector_data->total_buffered_metrics in the worker to show the statistics.
411
+ size_t buffered_metrics = (size_t)stats->buffered_metrics;
412
+ stats->buffered_metrics = 0;
413
+
414
+ size_t buffered_bytes = buffer_strlen(last_buffer->buffer);
415
+
416
+ last_buffer->buffered_metrics = buffered_metrics;
417
+ last_buffer->buffered_bytes = buffered_bytes;
418
+ last_buffer->used++;
419
+
420
+ simple_connector_data->total_buffered_metrics += buffered_metrics;
421
+ stats->buffered_bytes += buffered_bytes;
422
+
423
+ simple_connector_data->last_buffer = simple_connector_data->last_buffer->next;
424
425
return 0;
426
}
exporting/prometheus/remote_write/README.md
+3
@@ -30,6 +30,9 @@ in the Netdata configuration directory and set the following options:
30
remote write URL path = /receive
31
```
32
33
+You can also add `:https` modifier to the connector type if you need to use the TLS/SSL protocol. For example:
34
+`remote_write:https:my_instance`.
35
+
36
`remote write URL path` is used to set an endpoint path for the remote write protocol. The default value is `/receive`.
37
For example, if your endpoint is `http://example.domain:example_port/storage/read`:
38
exporting/prometheus/remote_write/remote_write.c
+34
-35
@@ -10,28 +10,18 @@ char family[PROMETHEUS_ELEMENT_MAX + 1];
10
char units[PROMETHEUS_ELEMENT_MAX + 1] = "";
11
12
/**
13
- * Send header to a server
13
+ * Prepare HTTP header
14
*
15
- * @param sock a communication socket.
15
* @param instance an instance data structure.
17
- * @return Returns 0 on success, 1 on failure.
16
*/
19
-int prometheus_remote_write_send_header(int *sock, struct instance *instance)
17
+void prometheus_remote_write_prepare_header(struct instance *instance)
18
{
21
- int flags = 0;
22
-#ifdef MSG_NOSIGNAL
23
- flags += MSG_NOSIGNAL;
24
-#endif
25
-
19
struct prometheus_remote_write_specific_config *connector_specific_config =
20
instance->config.connector_specific_config;
28
-
29
- static BUFFER *header;
30
- if (!header)
31
- header = buffer_create(0);
21
+ struct simple_connector_data *simple_connector_data = instance->connector_specific_data;
22
23
buffer_sprintf(
34
- header,
24
+ simple_connector_data->last_buffer->header,
25
"POST %s HTTP/1.1\r\n"
26
"Host: %s\r\n"
27
"Accept: */*\r\n"
@@ -40,17 +30,7 @@ int prometheus_remote_write_send_header(int *sock, struct instance *instance)
30
"Content-Type: application/x-www-form-urlencoded\r\n\r\n",
31
connector_specific_config->remote_write_path,
32
instance->config.destination,
43
- buffer_strlen((BUFFER *)instance->buffer));
44
-
45
- size_t header_len = buffer_strlen(header);
46
- ssize_t written = send(*sock, buffer_tostring(header), header_len, flags);
47
-
48
- buffer_flush(header);
49
-
50
- if (written != -1 && (size_t)written == header_len)
51
- return 0;
52
- else
53
- return 1;
33
+ buffer_strlen(simple_connector_data->last_buffer->buffer));
34
}
35
36
/**
@@ -90,7 +70,8 @@ int process_prometheus_remote_write_response(BUFFER *buffer, struct instance *in
70
*/
71
void clean_prometheus_remote_write(struct instance *instance)
72
{
93
- freez(instance->connector_specific_data);
73
+ struct simple_connector_data *simple_connector_data = instance->connector_specific_data;
74
+ freez(simple_connector_data->connector_specific_data);
75
76
struct prometheus_remote_write_specific_config *connector_specific_config =
77
instance->config.connector_specific_config;
@@ -115,22 +96,32 @@ int init_prometheus_remote_write_instance(struct instance *instance)
96
instance->end_host_formatting = NULL;
97
instance->end_batch_formatting = format_batch_prometheus_remote_write;
98
118
- instance->send_header = prometheus_remote_write_send_header;
99
+ instance->prepare_header = prometheus_remote_write_prepare_header;
100
instance->check_response = process_prometheus_remote_write_response;
101
102
instance->buffer = (void *)buffer_create(0);
122
- if (!instance->buffer) {
123
- error("EXPORTING: cannot create buffer for AWS Kinesis exporting connector instance %s", instance->config.name);
124
- return 1;
125
- }
103
+
104
if (uv_mutex_init(&instance->mutex))
105
return 1;
106
if (uv_cond_init(&instance->cond_var))
107
return 1;
108
109
+ struct simple_connector_data *simple_connector_data = callocz(1, sizeof(struct simple_connector_data));
110
+ instance->connector_specific_data = simple_connector_data;
111
+
112
+#ifdef ENABLE_HTTPS
113
+ simple_connector_data->flags = NETDATA_SSL_START;
114
+ simple_connector_data->conn = NULL;
115
+ if (instance->config.options & EXPORTING_OPTION_USE_TLS) {
116
+ security_start_ssl(NETDATA_SSL_CONTEXT_EXPORTING);
117
+ }
118
+#endif
119
+
120
struct prometheus_remote_write_specific_data *connector_specific_data =
121
callocz(1, sizeof(struct prometheus_remote_write_specific_data));
133
- instance->connector_specific_data = (void *)connector_specific_data;
122
+ simple_connector_data->connector_specific_data = (void *)connector_specific_data;
123
+
124
+ simple_connector_init(instance);
125
126
connector_specific_data->write_request = init_write_request();
127
@@ -148,8 +139,10 @@ int init_prometheus_remote_write_instance(struct instance *instance)
139
*/
140
int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host)
141
{
142
+ struct simple_connector_data *simple_connector_data =
143
+ (struct simple_connector_data *)instance->connector_specific_data;
144
struct prometheus_remote_write_specific_data *connector_specific_data =
152
- (struct prometheus_remote_write_specific_data *)instance->connector_specific_data;
145
+ (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data;
146
147
char hostname[PROMETHEUS_ELEMENT_MAX + 1];
148
prometheus_label_copy(
@@ -225,8 +218,10 @@ int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st)
218
*/
219
int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *rd)
220
{
221
+ struct simple_connector_data *simple_connector_data =
222
+ (struct simple_connector_data *)instance->connector_specific_data;
223
struct prometheus_remote_write_specific_data *connector_specific_data =
229
- (struct prometheus_remote_write_specific_data *)instance->connector_specific_data;
224
+ (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data;
225
226
if (rd->collections_counter && !rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) {
227
char name[PROMETHEUS_LABELS_MAX + 1];
@@ -322,8 +317,10 @@ int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *
317
*/
318
int format_batch_prometheus_remote_write(struct instance *instance)
319
{
320
+ struct simple_connector_data *simple_connector_data =
321
+ (struct simple_connector_data *)instance->connector_specific_data;
322
struct prometheus_remote_write_specific_data *connector_specific_data =
326
- (struct prometheus_remote_write_specific_data *)instance->connector_specific_data;
323
+ (struct prometheus_remote_write_specific_data *)simple_connector_data->connector_specific_data;
324
325
size_t data_size = get_write_request_size(connector_specific_data->write_request);
326
@@ -342,5 +339,7 @@ int format_batch_prometheus_remote_write(struct instance *instance)
339
buffer->len = data_size;
340
instance->stats.buffered_bytes = (collected_number)buffer_strlen(buffer);
341
342
+ simple_connector_end_batch(instance);
343
+
344
return 0;
345
}
exporting/prometheus/remote_write/remote_write.h
+5
-1
@@ -7,6 +7,10 @@
7
#include "exporting/prometheus/prometheus.h"
8
#include "remote_write_request.h"
9
10
+struct prometheus_remote_write_specific_data {
11
+ void *write_request;
12
+};
13
+
14
int init_prometheus_remote_write_instance(struct instance *instance);
15
extern void clean_prometheus_remote_write(struct instance *instance);
16
@@ -15,7 +19,7 @@ int format_chart_prometheus_remote_write(struct instance *instance, RRDSET *st);
19
int format_dimension_prometheus_remote_write(struct instance *instance, RRDDIM *rd);
20
int format_batch_prometheus_remote_write(struct instance *instance);
21
18
-int prometheus_remote_write_send_header(int *sock, struct instance *instance);
22
+void prometheus_remote_write_prepare_header(struct instance *instance);
23
int process_prometheus_remote_write_response(BUFFER *buffer, struct instance *instance);
24
25
#endif //NETDATA_EXPORTING_PROMETHEUS_REMOTE_WRITE_H
exporting/prometheus/remote_write/remote_write_request.h
-4
@@ -7,10 +7,6 @@
7
extern "C" {
8
#endif
9
10
-struct prometheus_remote_write_specific_data {
11
- void *write_request;
12
-};
13
-
10
void *init_write_request();
11
12
void add_host_info(
exporting/pubsub/pubsub.c
+1
-1
@@ -26,7 +26,7 @@ int init_pubsub_instance(struct instance *instance)
26
instance->end_host_formatting = flush_host_labels;
27
instance->end_batch_formatting = NULL;
28
29
- instance->send_header = NULL;
29
+ instance->prepare_header = NULL;
30
instance->check_response = NULL;
31
32
instance->buffer = (void *)buffer_create(0);
exporting/read_config.c
+28
-6
@@ -135,13 +135,20 @@ EXPORTING_CONNECTOR_TYPE exporting_select_type(const char *type)
135
{
136
if (!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
137
return EXPORTING_CONNECTOR_TYPE_GRAPHITE;
138
- } else if (!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
139
- return EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_TELNET;
140
- } else if (!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) {
141
- return EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP;
138
+ } else if (!strcmp(type, "graphite:http") || !strcmp(type, "graphite:https")) {
139
+ return EXPORTING_CONNECTOR_TYPE_GRAPHITE_HTTP;
140
} else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
141
return EXPORTING_CONNECTOR_TYPE_JSON;
144
- } else if (!strcmp(type, "prometheus_remote_write")) {
142
+ } else if (!strcmp(type, "json:http") || !strcmp(type, "json:https")) {
143
+ return EXPORTING_CONNECTOR_TYPE_JSON_HTTP;
144
+ } else if (!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
145
+ return EXPORTING_CONNECTOR_TYPE_OPENTSDB;
146
+ } else if (!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) {
147
+ return EXPORTING_CONNECTOR_TYPE_OPENTSDB_HTTP;
148
+ } else if (
149
+ !strcmp(type, "prometheus_remote_write") ||
150
+ !strcmp(type, "prometheus_remote_write:http") ||
151
+ !strcmp(type, "prometheus_remote_write:https")) {
152
return EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE;
153
} else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
154
return EXPORTING_CONNECTOR_TYPE_KINESIS;
@@ -432,7 +439,22 @@ struct engine *read_exporting_config()
439
tmp_instance->config.prefix = strdupz(exporter_get(instance_name, "prefix", "netdata"));
440
441
#ifdef ENABLE_HTTPS
435
- if (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP && !strncmp(tmp_ci_list->local_ci.connector_name, "opentsdb:https", 14)) {
442
+
443
+#define STR_GRAPHITE_HTTPS "graphite:https"
444
+#define STR_JSON_HTTPS "json:https"
445
+#define STR_OPENTSDB_HTTPS "opentsdb:https"
446
+#define STR_PROMETHEUS_REMOTE_WRITE_HTTPS "prometheus_remote_write:https"
447
+
448
+ if ((tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_GRAPHITE_HTTP &&
449
+ !strncmp(tmp_ci_list->local_ci.connector_name, STR_GRAPHITE_HTTPS, strlen(STR_GRAPHITE_HTTPS))) ||
450
+ (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_JSON_HTTP &&
451
+ !strncmp(tmp_ci_list->local_ci.connector_name, STR_JSON_HTTPS, strlen(STR_JSON_HTTPS))) ||
452
+ (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_OPENTSDB_HTTP &&
453
+ !strncmp(tmp_ci_list->local_ci.connector_name, STR_OPENTSDB_HTTPS, strlen(STR_OPENTSDB_HTTPS))) ||
454
+ (tmp_instance->config.type == EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE &&
455
+ !strncmp(
456
+ tmp_ci_list->local_ci.connector_name, STR_PROMETHEUS_REMOTE_WRITE_HTTPS,
457
+ strlen(STR_PROMETHEUS_REMOTE_WRITE_HTTPS)))) {
458
tmp_instance->config.options |= EXPORTING_OPTION_USE_TLS;
459
}
460
#endif
exporting/send_data.c
+160
-118
@@ -2,6 +2,22 @@
2
3
#include "exporting_engine.h"
4
5
+/**
6
+ * Check if TLS is enabled in the configuration
7
+ *
8
+ * @param type buffer with response data.
9
+ * @param options an instance data structure.
10
+ * @return Returns 1 if TLS should be enabled, 0 otherwise.
11
+ */
12
+static int exporting_tls_is_enabled(EXPORTING_CONNECTOR_TYPE type, EXPORTING_OPTIONS options)
13
+{
14
+ return (type == EXPORTING_CONNECTOR_TYPE_GRAPHITE_HTTP ||
15
+ type == EXPORTING_CONNECTOR_TYPE_JSON_HTTP ||
16
+ type == EXPORTING_CONNECTOR_TYPE_OPENTSDB_HTTP ||
17
+ type == EXPORTING_CONNECTOR_TYPE_PROMETHEUS_REMOTE_WRITE) &&
18
+ options & EXPORTING_OPTION_USE_TLS;
19
+}
20
+
21
/**
22
* Discard response
23
*
@@ -12,6 +28,7 @@
28
* @return Always returns 0.
29
*/
30
int exporting_discard_response(BUFFER *buffer, struct instance *instance) {
31
+#if NETDATA_INTERNAL_CHECKS
32
char sample[1024];
33
const char *s = buffer_tostring(buffer);
34
char *d = sample, *e = &sample[sizeof(sample) - 1];
@@ -23,11 +40,16 @@ int exporting_discard_response(BUFFER *buffer, struct instance *instance) {
40
}
41
*d = '\0';
42
26
- info(
43
+ debug(
44
+ D_BACKEND,
45
"EXPORTING: received %zu bytes from %s connector instance. Ignoring them. Sample: '%s'",
46
buffer_strlen(buffer),
47
instance->config.name,
48
sample);
49
+#else
50
+ UNUSED(instance);
51
+#endif /* NETDATA_INTERNAL_CHECKS */
52
+
53
buffer_flush(buffer);
54
return 0;
55
}
@@ -47,7 +69,7 @@ void simple_connector_receive_response(int *sock, struct instance *instance)
69
struct stats *stats = &instance->stats;
70
#ifdef ENABLE_HTTPS
71
uint32_t options = (uint32_t)instance->config.options;
50
- struct opentsdb_specific_data *connector_specific_data = instance->connector_specific_data;
72
+ struct simple_connector_data *connector_specific_data = instance->connector_specific_data;
73
74
if (options & EXPORTING_OPTION_USE_TLS)
75
ERR_clear_error();
@@ -61,8 +83,7 @@ void simple_connector_receive_response(int *sock, struct instance *instance)
83
84
ssize_t r;
85
#ifdef ENABLE_HTTPS
64
- if (instance->config.type == EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP &&
65
- options & EXPORTING_OPTION_USE_TLS &&
86
+ if (exporting_tls_is_enabled(instance->config.type, options) &&
87
connector_specific_data->conn &&
88
connector_specific_data->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) {
89
r = (ssize_t)SSL_read(connector_specific_data->conn,
@@ -132,11 +153,9 @@ endloop:
153
* @param failures the number of communication failures.
154
* @param instance an instance data structure.
155
*/
135
-void simple_connector_send_buffer(int *sock, int *failures, struct instance *instance)
156
+void simple_connector_send_buffer(
157
+ int *sock, int *failures, struct instance *instance, BUFFER *header, BUFFER *buffer, size_t buffered_metrics)
158
{
137
- BUFFER *buffer = (BUFFER *)instance->buffer;
138
- size_t len = buffer_strlen(buffer);
139
-
159
int flags = 0;
160
#ifdef MSG_NOSIGNAL
161
flags += MSG_NOSIGNAL;
@@ -144,58 +163,61 @@ void simple_connector_send_buffer(int *sock, int *failures, struct instance *ins
163
164
#ifdef ENABLE_HTTPS
165
uint32_t options = (uint32_t)instance->config.options;
147
- struct opentsdb_specific_data *connector_specific_data = instance->connector_specific_data;
166
+ struct simple_connector_data *connector_specific_data = instance->connector_specific_data;
167
168
if (options & EXPORTING_OPTION_USE_TLS)
169
ERR_clear_error();
170
#endif
171
172
struct stats *stats = &instance->stats;
173
+ ssize_t header_sent_bytes = 0;
174
+ ssize_t buffer_sent_bytes = 0;
175
+ size_t header_len = buffer_strlen(header);
176
+ size_t buffer_len = buffer_strlen(buffer);
177
155
- int ret = 0;
156
- if (instance->send_header)
157
- ret = instance->send_header(sock, instance);
158
-
159
- ssize_t written = -1;
160
-
161
- if (!ret) {
178
#ifdef ENABLE_HTTPS
163
- if (instance->config.type == EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP &&
164
- options & EXPORTING_OPTION_USE_TLS &&
165
- connector_specific_data->conn &&
166
- connector_specific_data->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) {
167
- written = (ssize_t)SSL_write(connector_specific_data->conn, buffer_tostring(buffer), len);
168
- } else {
169
- written = send(*sock, buffer_tostring(buffer), len, flags);
170
- }
179
+ if (exporting_tls_is_enabled(instance->config.type, options) &&
180
+ connector_specific_data->conn &&
181
+ connector_specific_data->flags == NETDATA_SSL_HANDSHAKE_COMPLETE) {
182
+ if (header_len)
183
+ header_sent_bytes = (ssize_t)SSL_write(connector_specific_data->conn, buffer_tostring(header), header_len);
184
+ if ((size_t)header_sent_bytes == header_len)
185
+ buffer_sent_bytes = (ssize_t)SSL_write(connector_specific_data->conn, buffer_tostring(buffer), buffer_len);
186
+ } else {
187
+ if (header_len)
188
+ header_sent_bytes = send(*sock, buffer_tostring(header), header_len, flags);
189
+ if ((size_t)header_sent_bytes == header_len)
190
+ buffer_sent_bytes = send(*sock, buffer_tostring(buffer), buffer_len, flags);
191
+ }
192
#else
172
- written = send(*sock, buffer_tostring(buffer), len, flags);
193
+ if (header_len)
194
+ header_sent_bytes = send(*sock, buffer_tostring(header), header_len, flags);
195
+ if ((size_t)header_sent_bytes == header_len)
196
+ buffer_sent_bytes = send(*sock, buffer_tostring(buffer), buffer_len, flags);
197
#endif
174
- }
198
176
- if(written != -1 && (size_t)written == len) {
199
+ if ((size_t)buffer_sent_bytes == buffer_len) {
200
// we sent the data successfully
201
stats->transmission_successes++;
179
- stats->sent_bytes += written;
180
- stats->sent_metrics = stats->buffered_metrics;
202
+ stats->sent_metrics += buffered_metrics;
203
+ stats->sent_bytes += buffer_sent_bytes;
204
205
// reset the failures count
206
*failures = 0;
207
208
// empty the buffer
209
buffer_flush(buffer);
187
- }
188
- else {
210
+ } else {
211
// oops! we couldn't send (all or some of the) data
212
error(
213
"EXPORTING: failed to write data to '%s'. Willing to write %zu bytes, wrote %zd bytes. Will re-connect.",
214
instance->config.destination,
193
- len,
194
- written);
215
+ buffer_len,
216
+ buffer_sent_bytes);
217
stats->transmission_failures++;
218
197
- if(written != -1)
198
- stats->sent_bytes += written;
219
+ if(buffer_sent_bytes != -1)
220
+ stats->sent_bytes += buffer_sent_bytes;
221
222
// increment the counter we check for data loss
223
(*failures)++;
@@ -206,22 +228,6 @@ void simple_connector_send_buffer(int *sock, int *failures, struct instance *ins
228
}
229
}
230
209
-/**
210
- * Clean up a simple connector instance on Netdata exit
211
- *
212
- * @param instance an instance data structure.
213
- */
214
-void simple_connector_cleanup(struct instance *instance)
215
-{
216
- info("EXPORTING: cleaning up instance %s ...", instance->config.name);
217
-
218
- buffer_free(instance->buffer);
219
- freez(instance->config.connector_specific_config);
220
-
221
- info("EXPORTING: instance %s exited", instance->config.name);
222
- instance->exited = 1;
223
-}
224
-
231
/**
232
* Simple connector worker
233
*
@@ -235,60 +241,97 @@ void simple_connector_worker(void *instance_p)
241
242
#ifdef ENABLE_HTTPS
243
uint32_t options = (uint32_t)instance->config.options;
238
- struct opentsdb_specific_data *connector_specific_data = instance->connector_specific_data;
244
+ struct simple_connector_data *connector_specific_data = instance->connector_specific_data;
245
246
if (options & EXPORTING_OPTION_USE_TLS)
247
ERR_clear_error();
248
#endif
249
struct simple_connector_config *connector_specific_config = instance->config.connector_specific_config;
244
- struct stats *stats = &instance->stats;
250
251
int sock = -1;
247
- struct timeval timeout = {.tv_sec = (instance->config.timeoutms * 1000) / 1000000,
248
- .tv_usec = (instance->config.timeoutms * 1000) % 1000000};
252
+ struct timeval timeout = { .tv_sec = (instance->config.timeoutms * 1000) / 1000000,
253
+ .tv_usec = (instance->config.timeoutms * 1000) % 1000000 };
254
int failures = 0;
255
251
- while(!instance->engine->exit) {
256
+ BUFFER *spare_header = buffer_create(0);
257
+ BUFFER *spare_buffer = buffer_create(0);
258
+
259
+ while (!instance->engine->exit) {
260
+ struct stats *stats = &instance->stats;
261
+ int send_stats = 0;
262
+
263
+ if (instance->data_is_ready)
264
+ send_stats = 1;
265
+
266
+ uv_mutex_lock(&instance->mutex);
267
+ if (!connector_specific_data->first_buffer->used || failures) {
268
+ while (!instance->data_is_ready)
269
+ uv_cond_wait(&instance->cond_var, &instance->mutex);
270
+ instance->data_is_ready = 0;
271
+ send_stats = 1;
272
+ }
273
+
274
+ if (unlikely(instance->engine->exit)) {
275
+ uv_mutex_unlock(&instance->mutex);
276
+ break;
277
+ }
278
+
279
+ // ------------------------------------------------------------------------
280
+ // detach buffer
281
+
282
+ BUFFER *header;
283
+ BUFFER *buffer;
284
+ size_t buffered_metrics;
285
+
286
+ if (!connector_specific_data->previous_buffer ||
287
+ (connector_specific_data->previous_buffer == connector_specific_data->first_buffer &&
288
+ connector_specific_data->first_buffer->used == 1)) {
289
+ connector_specific_data->header = connector_specific_data->first_buffer->header;
290
+ connector_specific_data->buffer = connector_specific_data->first_buffer->buffer;
291
+ connector_specific_data->buffered_metrics = connector_specific_data->first_buffer->buffered_metrics;
292
+ connector_specific_data->buffered_bytes = connector_specific_data->first_buffer->buffered_bytes;
293
+
294
+ header = connector_specific_data->header;
295
+ buffer = connector_specific_data->buffer;
296
+ buffered_metrics = connector_specific_data->buffered_metrics;
297
+
298
+ buffer_flush(spare_header);
299
+ connector_specific_data->first_buffer->header = spare_header;
300
+ spare_header = header;
301
+
302
+ buffer_flush(spare_buffer);
303
+ connector_specific_data->first_buffer->buffer = spare_buffer;
304
+ spare_buffer = buffer;
305
+ } else {
306
+ header = connector_specific_data->header;
307
+ buffer = connector_specific_data->buffer;
308
+ buffered_metrics = connector_specific_data->buffered_metrics;
309
+ }
310
253
- // reset the monitoring chart counters
254
- stats->received_bytes =
255
- stats->sent_bytes =
256
- stats->sent_metrics =
257
- stats->lost_metrics =
258
- stats->receptions =
259
- stats->transmission_successes =
260
- stats->transmission_failures =
261
- stats->data_lost_events =
262
- stats->lost_bytes =
263
- stats->reconnects = 0;
311
+ uv_mutex_unlock(&instance->mutex);
312
313
// ------------------------------------------------------------------------
314
// if we are connected, receive a response, without blocking
315
268
- if(likely(sock != -1))
316
+ if (likely(sock != -1))
317
simple_connector_receive_response(&sock, instance);
318
319
// ------------------------------------------------------------------------
320
// if we are not connected, connect to a data collecting server
321
274
- if(unlikely(sock == -1)) {
322
+ if (unlikely(sock == -1)) {
323
size_t reconnects = 0;
324
325
sock = connect_to_one_of(
278
- instance->config.destination,
279
- connector_specific_config->default_port,
280
- &timeout,
281
- &reconnects,
282
- NULL,
283
- 0);
326
+ instance->config.destination, connector_specific_config->default_port, &timeout, &reconnects, NULL, 0);
327
#ifdef ENABLE_HTTPS
285
- if(instance->config.type == EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP && sock != -1) {
286
- if (netdata_opentsdb_ctx) {
287
- if ( sock_delnonblock(sock) < 0 )
328
+ if (exporting_tls_is_enabled(instance->config.type, options) && sock != -1) {
329
+ if (netdata_exporting_ctx) {
330
+ if (sock_delnonblock(sock) < 0)
331
error("Exporting cannot remove the non-blocking flag from socket %d", sock);
332
333
if (connector_specific_data->conn == NULL) {
291
- connector_specific_data->conn = SSL_new(netdata_opentsdb_ctx);
334
+ connector_specific_data->conn = SSL_new(netdata_exporting_ctx);
335
if (connector_specific_data->conn == NULL) {
336
error("Failed to allocate SSL structure to socket %d.", sock);
337
connector_specific_data->flags = NETDATA_SSL_NO_HANDSHAKE;
@@ -307,50 +350,42 @@ void simple_connector_worker(void *instance_p)
350
int err = SSL_connect(connector_specific_data->conn);
351
if (err != 1) {
352
err = SSL_get_error(connector_specific_data->conn, err);
310
- error("SSL cannot connect with the server: %s ",
311
- ERR_error_string((long)SSL_get_error(connector_specific_data->conn, err), NULL));
353
+ error(
354
+ "SSL cannot connect with the server: %s ",
355
+ ERR_error_string((long)SSL_get_error(connector_specific_data->conn, err), NULL));
356
connector_specific_data->flags = NETDATA_SSL_NO_HANDSHAKE;
357
} else {
358
info("Exporting established a SSL connection.");
359
360
struct timeval tv;
317
- tv.tv_sec = timeout.tv_sec /4;
361
+ tv.tv_sec = timeout.tv_sec / 4;
362
tv.tv_usec = 0;
363
364
if (!tv.tv_sec)
365
tv.tv_sec = 2;
366
323
- if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof(tv)))
367
+ if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, (const char *)&tv, sizeof(tv)))
368
error("Cannot set timeout to socket %d, this can block communication", sock);
325
-
369
}
370
}
371
}
372
}
330
-
373
}
374
#endif
375
376
stats->reconnects += reconnects;
377
}
378
337
- if(unlikely(instance->engine->exit)) break;
379
+ if (unlikely(instance->engine->exit))
380
+ break;
381
382
// ------------------------------------------------------------------------
383
// if we are connected, send our buffer to the data collecting server
384
342
- uv_mutex_lock(&instance->mutex);
343
- while (!instance->data_is_ready)
344
- uv_cond_wait(&instance->cond_var, &instance->mutex);
345
- instance->data_is_ready = 0;
346
-
347
- if (unlikely(instance->engine->exit)) {
348
- uv_mutex_unlock(&instance->mutex);
349
- break;
350
- }
385
+ failures = 0;
386
387
if (likely(sock != -1)) {
353
- simple_connector_send_buffer(&sock, &failures, instance);
388
+ simple_connector_send_buffer(&sock, &failures, instance, header, buffer, buffered_metrics);
389
} else {
390
error("EXPORTING: failed to update '%s'", instance->config.destination);
391
stats->transmission_failures++;
@@ -359,26 +394,40 @@ void simple_connector_worker(void *instance_p)
394
failures++;
395
}
396
362
- BUFFER *buffer = instance->buffer;
363
-
364
- if (failures > instance->config.buffer_on_failures) {
365
- stats->lost_bytes += buffer_strlen(buffer);
366
- error(
367
- "EXPORTING: connector instance %s reached %d exporting failures. "
368
- "Flushing buffers to protect this host - this results in data loss on server '%s'",
369
- instance->config.name, failures, instance->config.destination);
370
- buffer_flush(buffer);
371
- failures = 0;
372
- stats->data_lost_events++;
373
- stats->lost_metrics = stats->buffered_metrics;
397
+ if (!failures) {
398
+ connector_specific_data->first_buffer->buffered_metrics =
399
+ connector_specific_data->first_buffer->buffered_bytes = connector_specific_data->first_buffer->used = 0;
400
+ connector_specific_data->first_buffer = connector_specific_data->first_buffer->next;
401
}
402
376
- send_internal_metrics(instance);
403
+ if (unlikely(instance->engine->exit))
404
+ break;
405
+
406
+ if (send_stats) {
407
+ uv_mutex_lock(&instance->mutex);
408
+
409
+ stats->buffered_metrics = connector_specific_data->total_buffered_metrics;
410
+
411
+ send_internal_metrics(instance);
412
378
- if(likely(buffer_strlen(buffer) == 0))
413
stats->buffered_metrics = 0;
414
381
- uv_mutex_unlock(&instance->mutex);
415
+ // reset the internal monitoring chart counters
416
+ connector_specific_data->total_buffered_metrics =
417
+ stats->buffered_bytes =
418
+ stats->receptions =
419
+ stats->received_bytes =
420
+ stats->sent_metrics =
421
+ stats->sent_bytes =
422
+ stats->transmission_successes =
423
+ stats->transmission_failures =
424
+ stats->reconnects =
425
+ stats->data_lost_events =
426
+ stats->lost_metrics =
427
+ stats->lost_bytes = 0;
428
+
429
+ uv_mutex_unlock(&instance->mutex);
430
+ }
431
432
#ifdef UNIT_TESTING
433
return;
@@ -390,12 +439,5 @@ void simple_connector_worker(void *instance_p)
439
clean_prometheus_remote_write(instance);
440
#endif
441
393
-#ifdef ENABLE_HTTPS
394
- if (instance->config.type == EXPORTING_CONNECTOR_TYPE_OPENTSDB_USING_HTTP && options & EXPORTING_OPTION_USE_TLS) {
395
- SSL_free(connector_specific_data->conn);
396
- freez(instance->connector_specific_data);
397
- }
398
-#endif
399
-
442
simple_connector_cleanup(instance);
443
}
exporting/tests/exporting_doubles.c
+7
@@ -168,6 +168,13 @@ int __mock_end_batch_formatting(struct instance *instance)
168
return mock_type(int);
169
}
170
171
+int __wrap_simple_connector_end_batch(struct instance *instance)
172
+{
173
+ function_called();
174
+ check_expected_ptr(instance);
175
+ return mock_type(int);
176
+}
177
+
178
#if ENABLE_PROMETHEUS_REMOTE_WRITE
179
void *__wrap_init_write_request()
180
{
exporting/tests/test_exporting_engine.c
+83
-89
@@ -122,7 +122,6 @@ static void test_init_connectors(void **state)
122
assert_ptr_equal(instance->metric_formatting, format_dimension_collected_graphite_plaintext);
123
assert_ptr_equal(instance->end_chart_formatting, NULL);
124
assert_ptr_equal(instance->end_host_formatting, flush_host_labels);
125
- assert_ptr_equal(instance->end_batch_formatting, simple_connector_update_buffered_bytes);
125
126
BUFFER *buffer = instance->buffer;
127
assert_ptr_not_equal(buffer, NULL);
@@ -500,14 +499,10 @@ static void test_format_dimension_collected_opentsdb_http(void **state)
499
assert_int_equal(format_dimension_collected_opentsdb_http(engine->instance_root, rd), 0);
500
assert_string_equal(
501
buffer_tostring(engine->instance_root->buffer),
503
- "POST /api/put HTTP/1.1\r\n"
504
- "Host: test-host\r\n"
505
- "Content-Type: application/json\r\n"
506
- "Content-Length: 153\r\n\r\n"
507
- "{ \"metric\": \"netdata.chart_name.dimension_name\", "
508
- "\"timestamp\": 15051, "
509
- "\"value\": 123000321, "
510
- "\"tags\": { \"host\": \"test-host TAG1=VALUE1 TAG2=VALUE2\" }}");
502
+ "{\"metric\":\"netdata.chart_name.dimension_name\","
503
+ "\"timestamp\":15051,"
504
+ "\"value\":123000321,"
505
+ "\"tags\":{\"host\":\"test-host TAG1=VALUE1 TAG2=VALUE2\"}}");
506
}
507
508
static void test_format_dimension_stored_opentsdb_http(void **state)
@@ -521,14 +516,10 @@ static void test_format_dimension_stored_opentsdb_http(void **state)
516
assert_int_equal(format_dimension_stored_opentsdb_http(engine->instance_root, rd), 0);
517
assert_string_equal(
518
buffer_tostring(engine->instance_root->buffer),
524
- "POST /api/put HTTP/1.1\r\n"
525
- "Host: test-host\r\n"
526
- "Content-Type: application/json\r\n"
527
- "Content-Length: 161\r\n\r\n"
528
- "{ \"metric\": \"netdata.chart_name.dimension_name\", "
529
- "\"timestamp\": 15052, "
530
- "\"value\": 690565856.0000000, "
531
- "\"tags\": { \"host\": \"test-host TAG1=VALUE1 TAG2=VALUE2\" }}");
519
+ "{\"metric\":\"netdata.chart_name.dimension_name\","
520
+ "\"timestamp\":15052,"
521
+ "\"value\":690565856.0000000,"
522
+ "\"tags\":{\"host\":\"test-host TAG1=VALUE1 TAG2=VALUE2\"}}");
523
}
524
525
static void test_exporting_discard_response(void **state)
@@ -538,14 +529,7 @@ static void test_exporting_discard_response(void **state)
529
BUFFER *response = buffer_create(0);
530
buffer_sprintf(response, "Test response");
531
541
- expect_function_call(__wrap_info_int);
542
-
532
assert_int_equal(exporting_discard_response(response, engine->instance_root), 0);
544
-
545
- assert_string_equal(
546
- log_line,
547
- "EXPORTING: received 13 bytes from instance_name connector instance. Ignoring them. Sample: 'Test response'");
548
-
533
assert_int_equal(buffer_strlen(response), 0);
534
535
buffer_free(response);
@@ -565,14 +549,8 @@ static void test_simple_connector_receive_response(void **state)
549
expect_value(__wrap_recv, len, 4096);
550
expect_value(__wrap_recv, flags, MSG_DONTWAIT);
551
568
- expect_function_call(__wrap_info_int);
569
-
552
simple_connector_receive_response(&sock, instance);
553
572
- assert_string_equal(
573
- log_line,
574
- "EXPORTING: received 9 bytes from instance_name connector instance. Ignoring them. Sample: 'Test recv'");
575
-
554
assert_int_equal(stats->received_bytes, 9);
555
assert_int_equal(stats->receptions, 1);
556
assert_int_equal(sock, 1);
@@ -583,39 +561,34 @@ static void test_simple_connector_send_buffer(void **state)
561
struct engine *engine = *state;
562
struct instance *instance = engine->instance_root;
563
struct stats *stats = &instance->stats;
586
- BUFFER *buffer = instance->buffer;
564
565
int sock = 1;
566
int failures = 3;
567
+ size_t buffered_metrics = 1;
568
+ BUFFER *header = buffer_create(0);
569
+ BUFFER *buffer = buffer_create(0);
570
+ buffer_strcat(header, "test header\n");
571
+ buffer_strcat(buffer, "test buffer\n");
572
591
- __real_mark_scheduled_instances(engine);
592
-
593
- expect_function_call(__wrap_rrdhost_is_exportable);
594
- expect_value(__wrap_rrdhost_is_exportable, instance, instance);
595
- expect_value(__wrap_rrdhost_is_exportable, host, localhost);
596
- will_return(__wrap_rrdhost_is_exportable, 1);
597
-
598
- RRDSET *st = localhost->rrdset_root;
599
- expect_function_call(__wrap_rrdset_is_exportable);
600
- expect_value(__wrap_rrdset_is_exportable, instance, instance);
601
- expect_value(__wrap_rrdset_is_exportable, st, st);
602
- will_return(__wrap_rrdset_is_exportable, 1);
603
-
604
- __real_prepare_buffers(engine);
573
+ expect_function_call(__wrap_send);
574
+ expect_value(__wrap_send, sockfd, 1);
575
+ expect_value(__wrap_send, buf, buffer_tostring(header));
576
+ expect_string(__wrap_send, buf, "test header\n");
577
+ expect_value(__wrap_send, len, 12);
578
+ expect_value(__wrap_send, flags, MSG_NOSIGNAL);
579
580
expect_function_call(__wrap_send);
581
expect_value(__wrap_send, sockfd, 1);
582
expect_value(__wrap_send, buf, buffer_tostring(buffer));
609
- expect_string(
610
- __wrap_send, buf, "netdata.test-host.chart_name.dimension_name;TAG1=VALUE1 TAG2=VALUE2 123000321 15051\n");
611
- expect_value(__wrap_send, len, 84);
583
+ expect_string(__wrap_send, buf, "test buffer\n");
584
+ expect_value(__wrap_send, len, 12);
585
expect_value(__wrap_send, flags, MSG_NOSIGNAL);
586
614
- simple_connector_send_buffer(&sock, &failures, instance);
587
+ simple_connector_send_buffer(&sock, &failures, instance, header, buffer, buffered_metrics);
588
589
assert_int_equal(failures, 0);
590
assert_int_equal(stats->transmission_successes, 1);
618
- assert_int_equal(stats->sent_bytes, 84);
591
+ assert_int_equal(stats->sent_bytes, 12);
592
assert_int_equal(stats->sent_metrics, 1);
593
assert_int_equal(stats->transmission_failures, 0);
594
@@ -629,22 +602,18 @@ static void test_simple_connector_worker(void **state)
602
struct engine *engine = *state;
603
struct instance *instance = engine->instance_root;
604
struct stats *stats = &instance->stats;
632
- BUFFER *buffer = instance->buffer;
605
606
__real_mark_scheduled_instances(engine);
607
636
- expect_function_call(__wrap_rrdhost_is_exportable);
637
- expect_value(__wrap_rrdhost_is_exportable, instance, instance);
638
- expect_value(__wrap_rrdhost_is_exportable, host, localhost);
639
- will_return(__wrap_rrdhost_is_exportable, 1);
608
+ struct simple_connector_data *simple_connector_data = callocz(1, sizeof(struct simple_connector_data));
609
+ instance->connector_specific_data = simple_connector_data;
610
+ simple_connector_data->last_buffer = callocz(1, sizeof(struct simple_connector_buffer));
611
+ simple_connector_data->first_buffer = simple_connector_data->last_buffer;
612
+ simple_connector_data->last_buffer->header = buffer_create(0);
613
+ simple_connector_data->last_buffer->buffer = buffer_create(0);
614
641
- RRDSET *st = localhost->rrdset_root;
642
- expect_function_call(__wrap_rrdset_is_exportable);
643
- expect_value(__wrap_rrdset_is_exportable, instance, instance);
644
- expect_value(__wrap_rrdset_is_exportable, st, st);
645
- will_return(__wrap_rrdset_is_exportable, 1);
646
-
647
- __real_prepare_buffers(engine);
615
+ buffer_sprintf(simple_connector_data->last_buffer->header, "test header");
616
+ buffer_sprintf(simple_connector_data->last_buffer->buffer, "test buffer");
617
618
expect_function_call(__wrap_connect_to_one_of);
619
expect_string(__wrap_connect_to_one_of, destination, "localhost");
@@ -656,10 +625,16 @@ static void test_simple_connector_worker(void **state)
625
626
expect_function_call(__wrap_send);
627
expect_value(__wrap_send, sockfd, 2);
659
- expect_value(__wrap_send, buf, buffer_tostring(buffer));
660
- expect_string(
661
- __wrap_send, buf, "netdata.test-host.chart_name.dimension_name;TAG1=VALUE1 TAG2=VALUE2 123000321 15051\n");
662
- expect_value(__wrap_send, len, 84);
628
+ expect_not_value(__wrap_send, buf, buffer_tostring(simple_connector_data->last_buffer->buffer));
629
+ expect_string(__wrap_send, buf, "test header");
630
+ expect_value(__wrap_send, len, 11);
631
+ expect_value(__wrap_send, flags, MSG_NOSIGNAL);
632
+
633
+ expect_function_call(__wrap_send);
634
+ expect_value(__wrap_send, sockfd, 2);
635
+ expect_value(__wrap_send, buf, buffer_tostring(simple_connector_data->last_buffer->buffer));
636
+ expect_string(__wrap_send, buf, "test buffer");
637
+ expect_value(__wrap_send, len, 11);
638
expect_value(__wrap_send, flags, MSG_NOSIGNAL);
639
640
expect_function_call(__wrap_send_internal_metrics);
@@ -669,13 +644,13 @@ static void test_simple_connector_worker(void **state)
644
simple_connector_worker(instance);
645
646
assert_int_equal(stats->buffered_metrics, 0);
672
- assert_int_equal(stats->buffered_bytes, 84);
647
+ assert_int_equal(stats->buffered_bytes, 0);
648
assert_int_equal(stats->received_bytes, 0);
674
- assert_int_equal(stats->sent_bytes, 84);
675
- assert_int_equal(stats->sent_metrics, 1);
649
+ assert_int_equal(stats->sent_bytes, 0);
650
+ assert_int_equal(stats->sent_metrics, 0);
651
assert_int_equal(stats->lost_metrics, 0);
652
assert_int_equal(stats->receptions, 0);
678
- assert_int_equal(stats->transmission_successes, 1);
653
+ assert_int_equal(stats->transmission_successes, 0);
654
assert_int_equal(stats->transmission_failures, 0);
655
assert_int_equal(stats->data_lost_events, 0);
656
assert_int_equal(stats->lost_bytes, 0);
@@ -1159,7 +1134,7 @@ static void test_init_prometheus_remote_write_instance(void **state)
1134
assert_ptr_equal(instance->end_chart_formatting, NULL);
1135
assert_ptr_equal(instance->end_host_formatting, NULL);
1136
assert_ptr_equal(instance->end_batch_formatting, format_batch_prometheus_remote_write);
1162
- assert_ptr_equal(instance->send_header, prometheus_remote_write_send_header);
1137
+ assert_ptr_equal(instance->prepare_header, prometheus_remote_write_prepare_header);
1138
assert_ptr_equal(instance->check_response, process_prometheus_remote_write_response);
1139
1140
assert_ptr_not_equal(instance->buffer, NULL);
@@ -1169,40 +1144,43 @@ static void test_init_prometheus_remote_write_instance(void **state)
1144
(struct prometheus_remote_write_specific_data *)instance->connector_specific_data;
1145
1146
assert_ptr_not_equal(instance->connector_specific_data, NULL);
1172
- assert_ptr_equal(connector_specific_data->write_request, 0xff);
1147
+ assert_ptr_not_equal(connector_specific_data->write_request, NULL);
1148
freez(instance->connector_specific_data);
1149
}
1150
1176
-static void test_prometheus_remote_write_send_header(void **state)
1151
+static void test_prometheus_remote_write_prepare_header(void **state)
1152
{
1153
struct engine *engine = *state;
1154
struct instance *instance = engine->instance_root;
1180
- int sock = 1;
1155
1156
struct prometheus_remote_write_specific_config *connector_specific_config =
1157
callocz(1, sizeof(struct prometheus_remote_write_specific_config));
1158
instance->config.connector_specific_config = connector_specific_config;
1159
connector_specific_config->remote_write_path = strdupz("/receive");
1160
1187
- buffer_sprintf(instance->buffer, "test buffer");
1161
+ struct simple_connector_data *simple_connector_data = callocz(1, sizeof(struct simple_connector_data));
1162
+ instance->connector_specific_data = simple_connector_data;
1163
+ simple_connector_data->last_buffer = callocz(1, sizeof(struct simple_connector_buffer));
1164
+ simple_connector_data->last_buffer->header = buffer_create(0);
1165
+ simple_connector_data->last_buffer->buffer = buffer_create(0);
1166
1189
- expect_function_call(__wrap_send);
1190
- expect_value(__wrap_send, sockfd, 1);
1191
- expect_not_value(__wrap_send, buf, NULL);
1192
- expect_string(
1193
- __wrap_send, buf,
1167
+ buffer_sprintf(simple_connector_data->last_buffer->buffer, "test buffer");
1168
+
1169
+ prometheus_remote_write_prepare_header(instance);
1170
+
1171
+ assert_string_equal(
1172
+ buffer_tostring(simple_connector_data->last_buffer->header),
1173
"POST /receive HTTP/1.1\r\n"
1174
"Host: localhost\r\n"
1175
"Accept: */*\r\n"
1176
"X-Prometheus-Remote-Write-Version: 0.1.0\r\n"
1177
"Content-Length: 11\r\n"
1178
"Content-Type: application/x-www-form-urlencoded\r\n\r\n");
1200
- expect_value(__wrap_send, len, 167);
1201
- expect_value(__wrap_send, flags, MSG_NOSIGNAL);
1202
-
1203
- assert_int_equal(prometheus_remote_write_send_header(&sock, instance),0);
1179
1180
free(connector_specific_config->remote_write_path);
1181
+
1182
+ buffer_free(simple_connector_data->last_buffer->header);
1183
+ buffer_free(simple_connector_data->last_buffer->buffer);
1184
}
1185
1186
static void test_process_prometheus_remote_write_response(void **state)
@@ -1224,9 +1202,11 @@ static void test_format_host_prometheus_remote_write(void **state)
1202
instance->config.options |= EXPORTING_OPTION_SEND_CONFIGURED_LABELS;
1203
instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
1204
1205
+ struct simple_connector_data *simple_connector_data = mallocz(sizeof(struct simple_connector_data *));
1206
+ instance->connector_specific_data = simple_connector_data;
1207
struct prometheus_remote_write_specific_data *connector_specific_data =
1208
mallocz(sizeof(struct prometheus_remote_write_specific_data *));
1229
- instance->connector_specific_data = (void *)connector_specific_data;
1209
+ simple_connector_data->connector_specific_data = (void *)connector_specific_data;
1210
connector_specific_data->write_request = (void *)0xff;
1211
1212
localhost->program_name = strdupz("test_program");
@@ -1254,6 +1234,7 @@ static void test_format_host_prometheus_remote_write(void **state)
1234
assert_int_equal(format_host_prometheus_remote_write(instance, localhost), 0);
1235
1236
freez(connector_specific_data);
1237
+ freez(simple_connector_data);
1238
free(localhost->program_name);
1239
free(localhost->program_version);
1240
}
@@ -1263,9 +1244,11 @@ static void test_format_dimension_prometheus_remote_write(void **state)
1244
struct engine *engine = *state;
1245
struct instance *instance = engine->instance_root;
1246
1247
+ struct simple_connector_data *simple_connector_data = mallocz(sizeof(struct simple_connector_data *));
1248
+ instance->connector_specific_data = simple_connector_data;
1249
struct prometheus_remote_write_specific_data *connector_specific_data =
1250
mallocz(sizeof(struct prometheus_remote_write_specific_data *));
1268
- instance->connector_specific_data = (void *)connector_specific_data;
1251
+ simple_connector_data->connector_specific_data = (void *)connector_specific_data;
1252
connector_specific_data->write_request = (void *)0xff;
1253
1254
RRDDIM *rd = localhost->rrdset_root->dimensions;
@@ -1291,11 +1274,16 @@ static void test_format_batch_prometheus_remote_write(void **state)
1274
struct engine *engine = *state;
1275
struct instance *instance = engine->instance_root;
1276
1277
+ struct simple_connector_data *simple_connector_data = mallocz(sizeof(struct simple_connector_data *));
1278
+ instance->connector_specific_data = simple_connector_data;
1279
struct prometheus_remote_write_specific_data *connector_specific_data =
1280
mallocz(sizeof(struct prometheus_remote_write_specific_data *));
1296
- instance->connector_specific_data = (void *)connector_specific_data;
1281
+ simple_connector_data->connector_specific_data = (void *)connector_specific_data;
1282
connector_specific_data->write_request = __real_init_write_request();
1283
1284
+ expect_function_call(__wrap_simple_connector_end_batch);
1285
+ expect_value(__wrap_simple_connector_end_batch, instance, instance);
1286
+ will_return(__wrap_simple_connector_end_batch, 0);
1287
__real_add_host_info(
1288
connector_specific_data->write_request,
1289
"test_name", "test_instance", "test_application", "test_version", 15051);
@@ -1410,6 +1398,9 @@ static void test_aws_kinesis_connector_worker(void **state)
1398
expect_value(__wrap_rrdset_is_exportable, st, st);
1399
will_return(__wrap_rrdset_is_exportable, 1);
1400
1401
+ expect_function_call(__wrap_simple_connector_end_batch);
1402
+ expect_value(__wrap_simple_connector_end_batch, instance, instance);
1403
+ will_return(__wrap_simple_connector_end_batch, 0);
1404
__real_prepare_buffers(engine);
1405
1406
struct aws_kinesis_specific_config *connector_specific_config =
@@ -1542,6 +1533,9 @@ static void test_pubsub_connector_worker(void **state)
1533
expect_value(__wrap_rrdset_is_exportable, st, st);
1534
will_return(__wrap_rrdset_is_exportable, 1);
1535
1536
+ expect_function_call(__wrap_simple_connector_end_batch);
1537
+ expect_value(__wrap_simple_connector_end_batch, instance, instance);
1538
+ will_return(__wrap_simple_connector_end_batch, 0);
1539
__real_prepare_buffers(engine);
1540
1541
struct pubsub_specific_config *connector_specific_config =
@@ -1663,7 +1657,7 @@ static void test_init_mongodb_instance(void **state)
1657
assert_ptr_equal(instance->end_chart_formatting, NULL);
1658
assert_ptr_equal(instance->end_host_formatting, flush_host_labels);
1659
assert_ptr_equal(instance->end_batch_formatting, format_batch_mongodb);
1666
- assert_ptr_equal(instance->send_header, NULL);
1660
+ assert_ptr_equal(instance->prepare_header, NULL);
1661
assert_ptr_equal(instance->check_response, NULL);
1662
1663
assert_ptr_not_equal(instance->buffer, NULL);
@@ -1884,7 +1878,7 @@ int main(void)
1878
cmocka_unit_test_setup_teardown(
1879
test_init_prometheus_remote_write_instance, setup_configured_engine, teardown_configured_engine),
1880
cmocka_unit_test_setup_teardown(
1887
- test_prometheus_remote_write_send_header, setup_initialized_engine, teardown_initialized_engine),
1881
+ test_prometheus_remote_write_prepare_header, setup_initialized_engine, teardown_initialized_engine),
1882
cmocka_unit_test(test_process_prometheus_remote_write_response),
1883
cmocka_unit_test_setup_teardown(
1884
test_format_host_prometheus_remote_write, setup_initialized_engine, teardown_initialized_engine),
exporting/tests/test_exporting_engine.h
+2
@@ -116,6 +116,8 @@ int __mock_end_chart_formatting(struct instance *instance, RRDSET *st);
116
int __mock_end_host_formatting(struct instance *instance, RRDHOST *host);
117
int __mock_end_batch_formatting(struct instance *instance);
118
119
+int __wrap_simple_connector_end_batch(struct instance *instance);
120
+
121
#if ENABLE_PROMETHEUS_REMOTE_WRITE
122
void *__real_init_write_request();
123
void *__wrap_init_write_request();
libnetdata/config/appconfig.c
+8
-2
@@ -69,14 +69,20 @@ int is_valid_connector(char *type, int check_reserved)
69
70
if (!strcmp(type, "graphite") || !strcmp(type, "graphite:plaintext")) {
71
return rc;
72
+ } else if (!strcmp(type, "graphite:http") || !strcmp(type, "graphite:https")) {
73
+ return rc;
74
+ } else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
75
+ return rc;
76
+ } else if (!strcmp(type, "json:http") || !strcmp(type, "json:https")) {
77
+ return rc;
78
} else if (!strcmp(type, "opentsdb") || !strcmp(type, "opentsdb:telnet")) {
79
return rc;
80
} else if (!strcmp(type, "opentsdb:http") || !strcmp(type, "opentsdb:https")) {
81
return rc;
76
- } else if (!strcmp(type, "json") || !strcmp(type, "json:plaintext")) {
77
- return rc;
82
} else if (!strcmp(type, "prometheus_remote_write")) {
83
return rc;
84
+ } else if (!strcmp(type, "prometheus_remote_write:http") || !strcmp(type, "prometheus_remote_write:https")) {
85
+ return rc;
86
} else if (!strcmp(type, "kinesis") || !strcmp(type, "kinesis:plaintext")) {
87
return rc;
88
} else if (!strcmp(type, "pubsub") || !strcmp(type, "pubsub:plaintext")) {
libnetdata/socket/security.c
+12
-14
@@ -2,7 +2,7 @@
2
3
#ifdef ENABLE_HTTPS
4
5
-SSL_CTX *netdata_opentsdb_ctx=NULL;
5
+SSL_CTX *netdata_exporting_ctx=NULL;
6
SSL_CTX *netdata_client_ctx=NULL;
7
SSL_CTX *netdata_srv_ctx=NULL;
8
const char *security_key=NULL;
@@ -201,7 +201,7 @@ static SSL_CTX * security_initialize_openssl_server() {
201
* @param selector informs the context that must be initialized, the following list has the valid values:
202
* NETDATA_SSL_CONTEXT_SERVER - the server context
203
* NETDATA_SSL_CONTEXT_STREAMING - Starts the streaming context.
204
- * NETDATA_SSL_CONTEXT_OPENTSDB - Starts the OpenTSDB contextv
204
+ * NETDATA_SSL_CONTEXT_EXPORTING - Starts the OpenTSDB contextv
205
*/
206
void security_start_ssl(int selector) {
207
switch (selector) {
@@ -222,8 +222,8 @@ void security_start_ssl(int selector) {
222
SSL_CTX_set_mode(netdata_client_ctx, SSL_MODE_ENABLE_PARTIAL_WRITE |SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER |SSL_MODE_AUTO_RETRY);
223
break;
224
}
225
- case NETDATA_SSL_CONTEXT_OPENTSDB: {
226
- netdata_opentsdb_ctx = security_initialize_openssl_client();
225
+ case NETDATA_SSL_CONTEXT_EXPORTING: {
226
+ netdata_exporting_ctx = security_initialize_openssl_client();
227
break;
228
}
229
}
@@ -234,20 +234,18 @@ void security_start_ssl(int selector) {
234
*
235
* Clean all the allocated contexts from netdata.
236
*/
237
-void security_clean_openssl() {
238
- if (netdata_srv_ctx)
239
- {
240
- SSL_CTX_free(netdata_srv_ctx);
241
- }
237
+void security_clean_openssl()
238
+{
239
+ if (netdata_srv_ctx) {
240
+ SSL_CTX_free(netdata_srv_ctx);
241
+ }
242
243
- if (netdata_client_ctx)
244
- {
243
+ if (netdata_client_ctx) {
244
SSL_CTX_free(netdata_client_ctx);
245
}
246
248
- if ( netdata_opentsdb_ctx )
249
- {
250
- SSL_CTX_free(netdata_opentsdb_ctx);
247
+ if (netdata_exporting_ctx) {
248
+ SSL_CTX_free(netdata_exporting_ctx);
249
}
250
251
#if OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110
libnetdata/socket/security.h
+2
-2
@@ -14,7 +14,7 @@
14
15
#define NETDATA_SSL_CONTEXT_SERVER 0
16
#define NETDATA_SSL_CONTEXT_STREAMING 1
17
-#define NETDATA_SSL_CONTEXT_OPENTSDB 2
17
+#define NETDATA_SSL_CONTEXT_EXPORTING 2
18
19
# ifdef ENABLE_HTTPS
20
@@ -34,7 +34,7 @@ struct netdata_ssl{
34
uint32_t flags; //The flags for SSL connection
35
};
36
37
-extern SSL_CTX *netdata_opentsdb_ctx;
37
+extern SSL_CTX *netdata_exporting_ctx;
38
extern SSL_CTX *netdata_client_ctx;
39
extern SSL_CTX *netdata_srv_ctx;
40
extern const char *security_key;
web/api/exporters/allmetrics.c
+15
-13
@@ -4,24 +4,26 @@
4
5
struct prometheus_output_options {
6
char *name;
7
- BACKENDS_PROMETHEUS_OUTPUT_OPTIONS flag;
7
+ PROMETHEUS_OUTPUT_OPTIONS flag;
8
} prometheus_output_flags_root[] = {
9
- { "help", BACKENDS_PROMETHEUS_OUTPUT_HELP },
10
- { "types", BACKENDS_PROMETHEUS_OUTPUT_TYPES },
11
- { "names", BACKENDS_PROMETHEUS_OUTPUT_NAMES },
12
- { "timestamps", BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS },
13
- { "variables", BACKENDS_PROMETHEUS_OUTPUT_VARIABLES },
14
- { "oldunits", BACKENDS_PROMETHEUS_OUTPUT_OLDUNITS },
15
- { "hideunits", BACKENDS_PROMETHEUS_OUTPUT_HIDEUNITS },
16
- // terminator
17
- { NULL, BACKENDS_PROMETHEUS_OUTPUT_NONE },
9
+ { "help", PROMETHEUS_OUTPUT_HELP },
10
+ { "types", PROMETHEUS_OUTPUT_TYPES },
11
+ { "names", PROMETHEUS_OUTPUT_NAMES },
12
+ { "timestamps", PROMETHEUS_OUTPUT_TIMESTAMPS },
13
+ { "variables", PROMETHEUS_OUTPUT_VARIABLES },
14
+ { "oldunits", PROMETHEUS_OUTPUT_OLDUNITS },
15
+ { "hideunits", PROMETHEUS_OUTPUT_HIDEUNITS },
16
+ // terminator
17
+ { NULL, PROMETHEUS_OUTPUT_NONE },
18
};
19
20
inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client *w, char *url) {
21
int format = ALLMETRICS_SHELL;
22
const char *prometheus_server = w->client_ip;
23
uint32_t prometheus_backend_options = global_backend_options;
24
- BACKENDS_PROMETHEUS_OUTPUT_OPTIONS prometheus_output_options = BACKENDS_PROMETHEUS_OUTPUT_TIMESTAMPS | ((global_backend_options & BACKEND_OPTION_SEND_NAMES)?BACKENDS_PROMETHEUS_OUTPUT_NAMES:0);
24
+ PROMETHEUS_OUTPUT_OPTIONS prometheus_output_options =
25
+ PROMETHEUS_OUTPUT_TIMESTAMPS |
26
+ ((global_backend_options & BACKEND_OPTION_SEND_NAMES) ? PROMETHEUS_OUTPUT_NAMES : 0);
27
const char *prometheus_prefix = global_backend_prefix;
28
29
while(url) {
@@ -84,7 +86,7 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
86
87
case ALLMETRICS_PROMETHEUS:
88
w->response.data->contenttype = CT_PROMETHEUS;
87
- backends_rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
89
+ rrd_stats_api_v1_charts_allmetrics_prometheus_single_host(
90
host
91
, w->response.data
92
, prometheus_server
@@ -96,7 +98,7 @@ inline int web_client_api_request_v1_allmetrics(RRDHOST *host, struct web_client
98
99
case ALLMETRICS_PROMETHEUS_ALL_HOSTS:
100
w->response.data->contenttype = CT_PROMETHEUS;
99
- backends_rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
101
+ rrd_stats_api_v1_charts_allmetrics_prometheus_all_hosts(
102
host
103
, w->response.data
104
, prometheus_server