Add http headers to responses (#8760)
The MQTT payloads for responses to API requests from the cloud now include a headers field with the raw http headers encoded into unicode. This exposes the `Date` and `Expired` fields to the cloud backend.
Andrew Moss committed
Apr 22, 2020 at 16:51 UTC
22f918af6e7bd1e8da57e3d2c543780554dc56e9
7 files changed
+73
-53
aclk/agent_cloud_link.c
+59
-43
@@ -723,6 +723,50 @@ void aclk_del_collector(const char *hostname, const char *plugin_name, const cha
723
}
724
725
_free_collector(tmp_collector);
726
+
727
+}
728
+/*
729
+ * Take a buffer, encode it and rewrite it
730
+ *
731
+ */
732
+
733
+static char *aclk_encode_response(char *src, size_t content_size, int keep_newlines)
734
+{
735
+ char *tmp_buffer = mallocz(content_size * 2);
736
+ char *dst = tmp_buffer;
737
+ while (content_size > 0) {
738
+ switch (*src) {
739
+ case '\n':
740
+ if (keep_newlines)
741
+ {
742
+ *dst++ = '\\';
743
+ *dst++ = 'n';
744
+ }
745
+ break;
746
+ case '\t':
747
+ break;
748
+ case 0x01 ... 0x08:
749
+ case 0x0b ... 0x1F:
750
+ *dst++ = '\\';
751
+ *dst++ = 'u';
752
+ *dst++ = '0';
753
+ *dst++ = '0';
754
+ *dst++ = (*src < 0x0F) ? '0' : '1';
755
+ *dst++ = to_hex(*src);
756
+ break;
757
+ case '\"':
758
+ *dst++ = '\\';
759
+ *dst++ = *src;
760
+ break;
761
+ default:
762
+ *dst++ = *src;
763
+ }
764
+ src++;
765
+ content_size--;
766
+ }
767
+ *dst = '\0';
768
+
769
+ return tmp_buffer;
770
}
771
772
int aclk_execute_query(struct aclk_query *this_query)
@@ -730,6 +774,8 @@ int aclk_execute_query(struct aclk_query *this_query)
774
if (strncmp(this_query->query, "/api/v1/", 8) == 0) {
775
struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client));
776
w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
777
+ w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
778
+ w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
779
strcpy(w->origin, "*"); // Simulate web_client_create_on_fd()
780
w->cookie1[0] = 0; // Simulate web_client_create_on_fd()
781
w->cookie2[0] = 0; // Simulate web_client_create_on_fd()
@@ -745,26 +791,36 @@ int aclk_execute_query(struct aclk_query *this_query)
791
mysep = strrchr(this_query->query, '/');
792
793
// TODO: handle bad response perhaps in a different way. For now it does to the payload
748
- int rc = web_client_api_request_v1(localhost, w, mysep ? mysep + 1 : "noop");
794
+ w->response.code = web_client_api_request_v1(localhost, w, mysep ? mysep + 1 : "noop");
795
+ now_realtime_timeval(&w->tv_ready);
796
+ w->response.data->date = w->tv_ready.tv_sec;
797
+ web_client_build_http_header(w); // TODO: this function should offset from date, not tv_ready
798
BUFFER *local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
799
buffer_flush(local_buffer);
800
local_buffer->contenttype = CT_APPLICATION_JSON;
801
802
aclk_create_header(local_buffer, "http", this_query->msg_id, 0, 0);
803
buffer_strcat(local_buffer, ",\n\t\"payload\": ");
755
- char *encoded_response = aclk_encode_response(w->response.data);
804
+ char *encoded_response = aclk_encode_response(w->response.data->buffer, w->response.data->len, 0);
805
+ char *encoded_header = aclk_encode_response(w->response.header_output->buffer, w->response.header_output->len, 1);
806
807
buffer_sprintf(
758
- local_buffer, "{\n\"code\": %d,\n\"body\": \"%s\"\n}", rc, encoded_response);
808
+ local_buffer, "{\n\"code\": %d,\n\"body\": \"%s\",\n\"headers\": \"%s\"\n}",
809
+ w->response.code, encoded_response, encoded_header);
810
811
buffer_sprintf(local_buffer, "\n}");
812
813
+ debug(D_ACLK, "Response:%s", encoded_header);
814
+
815
aclk_send_message(this_query->topic, local_buffer->buffer, this_query->msg_id);
816
817
buffer_free(w->response.data);
818
+ buffer_free(w->response.header);
819
+ buffer_free(w->response.header_output);
820
freez(w);
821
buffer_free(local_buffer);
822
freez(encoded_response);
823
+ freez(encoded_header);
824
return 0;
825
}
826
return 1;
@@ -1535,46 +1591,6 @@ inline void aclk_create_header(BUFFER *dest, char *type, char *msg_id, time_t ts
1591
debug(D_ACLK, "Sending v%d msgid [%s] type [%s] time [%ld]", ACLK_VERSION, msg_id, type, ts_secs);
1592
}
1593
1538
-/*
1539
- * Take a buffer, encode it and rewrite it
1540
- *
1541
- */
1542
-
1543
-char *aclk_encode_response(BUFFER *contents)
1544
-{
1545
- char *tmp_buffer = mallocz(contents->len * 2);
1546
- char *src, *dst;
1547
- size_t content_size = contents->len;
1548
-
1549
- src = contents->buffer;
1550
- dst = tmp_buffer;
1551
- while (content_size > 0) {
1552
- switch (*src) {
1553
- case '\n':
1554
- case '\t':
1555
- break;
1556
- case 0x01 ... 0x08:
1557
- case 0x0b ... 0x1F:
1558
- *dst++ = '\\';
1559
- *dst++ = '0';
1560
- *dst++ = '0';
1561
- *dst++ = (*src < 0x0F) ? '0' : '1';
1562
- *dst++ = to_hex(*src);
1563
- break;
1564
- case '\"':
1565
- *dst++ = '\\';
1566
- *dst++ = *src;
1567
- break;
1568
- default:
1569
- *dst++ = *src;
1570
- }
1571
- src++;
1572
- content_size--;
1573
- }
1574
- *dst = '\0';
1575
-
1576
- return tmp_buffer;
1577
-}
1594
1595
/*
1596
* This will send alarm information which includes
aclk/agent_cloud_link.h
-1
@@ -101,7 +101,6 @@ void aclk_del_collector(const char *hostname, const char *plugin_name, const cha
101
void aclk_alarm_reload();
102
void aclk_send_alarm_metadata();
103
int aclk_execute_query(struct aclk_query *query);
104
-char *aclk_encode_response(BUFFER *contents);
104
unsigned long int aclk_reconnect_delay(int mode);
105
extern void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host);
106
void aclk_single_update_enable();
build_external/bin/clean-install.sh
+5
-5
@@ -33,19 +33,19 @@ if cat <<HAPPY_CASE | grep "$DISTRO-$VERSION"
33
HAPPY_CASE
34
then
35
docker build -f "$BuildBase/clean-install.Dockerfile" -t "${DISTRO}_${VERSION}_dev" "$BuildBase/.." \
36
- --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" --build-arg ACLK=yes \
36
+ --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" \
37
--build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED"
38
else
39
case "$DISTRO-$VERSION" in
40
arch-current)
41
docker build -f "$BuildBase/clean-install-arch.Dockerfile" -t "${DISTRO}_${VERSION}_dev" "$BuildBase/.." \
42
- --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" --build-arg ACLK=yes \
43
- --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED"
42
+ --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" \
43
+ --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED" # --no-cache
44
;;
45
arch-extras) # Add valgrind to the container
46
docker build -f "$BuildBase/clean-install-arch-extras.Dockerfile" -t "${DISTRO}_${VERSION}_dev" "$BuildBase/.." \
47
- --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" --build-arg ACLK=yes \
48
- --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED"
47
+ --build-arg "DISTRO=$DISTRO" --build-arg "VERSION=$VERSION" \
48
+ --build-arg EXTRA_CFLAGS="-DACLK_SSL_ALLOW_SELF_SIGNED" # --no-cache
49
;;
50
*)
51
echo "Unknown $DISTRO-$VERSION"
build_external/clean-install-arch-extras.Dockerfile
+1
-2
@@ -22,7 +22,6 @@ RUN pacman --noconfirm --needed -S autoconf \
22
cmake \
23
valgrind
24
25
-ARG ACLK=no
25
ARG EXTRA_CFLAGS
26
COPY . /opt/netdata/source
27
WORKDIR /opt/netdata/source
@@ -46,7 +45,7 @@ RUN rm -rf .git/
45
RUN find . -type f >/opt/netdata/manifest
46
47
RUN CFLAGS="-O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1\
49
- -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --disable-lto
48
+ -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --require-cloud --disable-lto
49
50
RUN ln -sf /dev/stdout /var/log/netdata/access.log
51
RUN ln -sf /dev/stdout /var/log/netdata/debug.log
claim/netdata-claim.sh.in
+1
-1
@@ -101,7 +101,7 @@ TOKEN="unknown"
101
URL_BASE="https://netdata.cloud"
102
ID="unknown"
103
ROOMS=""
104
-HOSTNAME=$(hostname)
104
+[ -z "$HOSTNAME" ] && HOSTNAME=$(hostname)
105
CLOUD_CERTIFICATE_FILE="${CLAIMING_DIR}/cloud_fullchain.pem"
106
VERBOSE=0
107
INSECURE=0
web/server/web_client.c
+5
-1
@@ -1118,7 +1118,7 @@ static inline ssize_t web_client_send_data(struct web_client *w,const void *buf,
1118
return bytes;
1119
}
1120
1121
-static inline void web_client_send_http_header(struct web_client *w) {
1121
+void web_client_build_http_header(struct web_client *w) {
1122
if(unlikely(w->response.code != HTTP_RESP_OK))
1123
buffer_no_cacheable(w->response.data);
1124
@@ -1252,6 +1252,10 @@ static inline void web_client_send_http_header(struct web_client *w) {
1252
1253
// end of HTTP header
1254
buffer_strcat(w->response.header_output, "\r\n");
1255
+}
1256
+
1257
+static inline void web_client_send_http_header(struct web_client *w) {
1258
+ web_client_build_http_header(w);
1259
1260
// sent the HTTP header
1261
debug(D_WEB_DATA, "%llu: Sending response HTTP header of size %zu: '%s'"
web/server/web_client.h
+2
@@ -207,6 +207,8 @@ extern void buffer_data_options2string(BUFFER *wb, uint32_t options);
207
208
extern int mysendfile(struct web_client *w, char *filename);
209
210
+extern void web_client_build_http_header(struct web_client *w);
211
+
212
#include "daemon/common.h"
213
214
#endif