Fix outstanding problems in claiming and add SOCKS5 support. (#8406)
This commit fixes the known problems in claiming: incorrect reports of success, better treatment of error code and improved visibility of what the script is doing. There has been extensive testing against both environments to check that it works. The socks5 proxy support has been integrated and works for both methods of calling the claiming script. Co-authored-by: Timotej Šiškovič <timotej@netdata.cloud>
Andrew Moss committed
Mar 17, 2020 at 13:57 UTC
9e32f03a474cd37da288d43cf4df051d22e2e3d0
6 files changed
+202
-112
aclk/aclk_common.c
+42
@@ -105,3 +105,45 @@ const char *aclk_lws_wss_get_proxy_setting(ACLK_PROXY_TYPE *type) {
105
106
return proxy;
107
}
108
+
109
+int aclk_decode_base_url(char *url, char **aclk_hostname, char **aclk_port)
110
+{
111
+int pos = 0;
112
+ if (!strncmp("https://", url, 8))
113
+ {
114
+ pos = 8;
115
+ }
116
+ else if (!strncmp("http://", url, 7))
117
+ {
118
+ error("Cannot connect ACLK over %s -> unencrypted link is not supported", url);
119
+ return 1;
120
+ }
121
+int host_end = pos;
122
+ while( url[host_end] != 0 && url[host_end] != '/' && url[host_end] != ':' )
123
+ host_end++;
124
+ if (url[host_end] == 0)
125
+ {
126
+ *aclk_hostname = strdupz(url+pos);
127
+ *aclk_port = strdupz("443");
128
+ info("Setting ACLK target host=%s port=%s from %s", *aclk_hostname, *aclk_port, url);
129
+ return 0;
130
+ }
131
+ if (url[host_end] == ':')
132
+ {
133
+ *aclk_hostname = callocz(host_end - pos + 1, 1);
134
+ strncpy(*aclk_hostname, url+pos, host_end - pos);
135
+ int port_end = host_end + 1;
136
+ while (url[port_end] >= '0' && url[port_end] <= '9')
137
+ port_end++;
138
+ if (port_end - host_end > 6)
139
+ {
140
+ error("Port specified in %s is invalid", url);
141
+ return 0;
142
+ }
143
+ *aclk_port = callocz(port_end - host_end + 1, 1);
144
+ for(int i=host_end + 1; i < port_end; i++)
145
+ (*aclk_port)[i - host_end - 1] = url[i];
146
+ }
147
+ info("Setting ACLK target host=%s port=%s from %s", *aclk_hostname, *aclk_port, url);
148
+ return 0;
149
+}
aclk/agent_cloud_link.c
+2
-43
@@ -3,6 +3,7 @@
3
#include "libnetdata/libnetdata.h"
4
#include "agent_cloud_link.h"
5
#include "aclk_lws_https_client.h"
6
+#include "aclk_common.h"
7
8
// State-machine for the on-connect metadata transmission.
9
// TODO: The AGENT_STATE should be centralized as it would be useful to control error-logging during the initial
@@ -1129,48 +1130,6 @@ unsigned int line_len=0;
1130
return NULL;
1131
}
1132
1132
-static int decode_base_url(char *url, char **aclk_hostname, char **aclk_port)
1133
-{
1134
-int pos = 0;
1135
- if (!strncmp("https://", url, 8))
1136
- {
1137
- pos = 8;
1138
- }
1139
- else if (!strncmp("http://", url, 7))
1140
- {
1141
- error("Cannot connect ACLK over %s -> unencrypted link is not supported", url);
1142
- return 1;
1143
- }
1144
-int host_end = pos;
1145
- while( url[host_end] != 0 && url[host_end] != '/' && url[host_end] != ':' )
1146
- host_end++;
1147
- if (url[host_end] == 0)
1148
- {
1149
- *aclk_hostname = strdupz(url+pos);
1150
- *aclk_port = strdupz("443");
1151
- info("Setting ACLK target host=%s port=%s from %s", *aclk_hostname, *aclk_port, url);
1152
- return 0;
1153
- }
1154
- if (url[host_end] == ':')
1155
- {
1156
- *aclk_hostname = callocz(host_end - pos + 1, 1);
1157
- strncpy(*aclk_hostname, url+pos, host_end - pos);
1158
- int port_end = host_end + 1;
1159
- while (url[port_end] >= '0' && url[port_end] <= '9')
1160
- port_end++;
1161
- if (port_end - host_end > 6)
1162
- {
1163
- error("Port specified in %s is invalid", url);
1164
- return 0;
1165
- }
1166
- *aclk_port = callocz(port_end - host_end + 1, 1);
1167
- for(int i=host_end + 1; i < port_end; i++)
1168
- (*aclk_port)[i - host_end - 1] = url[i];
1169
- }
1170
- info("Setting ACLK target host=%s port=%s from %s", *aclk_hostname, *aclk_port, url);
1171
- return 0;
1172
-}
1173
-
1133
void aclk_get_challenge(char *aclk_hostname, char *aclk_port)
1134
{
1135
char *data_buffer = mallocz(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
@@ -1304,7 +1263,7 @@ void *aclk_main(void *ptr)
1263
char *aclk_port = NULL;
1264
uint32_t port_num = 0;
1265
char *cloud_base_url = config_get(CONFIG_SECTION_CLOUD, "cloud base url", "https://netdata.cloud");
1307
- if( decode_base_url(cloud_base_url, &aclk_hostname, &aclk_port))
1266
+ if( aclk_decode_base_url(cloud_base_url, &aclk_hostname, &aclk_port))
1267
{
1268
error("Configuration error - cannot use agent cloud link");
1269
return NULL;
aclk/tests/paho-inspection.py
+2
@@ -27,6 +27,8 @@ def on_message(mqttc, obj, msg):
27
print(f"Message {mtype} time={ts} size {len(api_msg)}", flush=True)
28
now = time.time()
29
print(f"Current {now} -> Delay {now-ts}", flush=True)
30
+ if mtype=="disconnect":
31
+ print(f"Message dump: {api_msg}", flush=True)
32
33
def on_publish(mqttc, obj, mid):
34
print("mid: "+str(mid), flush=True)
claim/README.md
+24
@@ -31,6 +31,8 @@ following arguments:
31
where AGENT_ID is the unique identifier of the agent. This is the agent's MACHINE_GUID by default.
32
-hostname=HOSTNAME
33
where HOSTNAME is the result of the hostname command by default.
34
+-proxy=PROXY_URL
35
+ where PROXY_URL is the endpoint of a SOCKS5 proxy.
36
```
37
38
For example, the following command claims an agent and adds it to rooms `room1` and `room2`:
@@ -76,4 +78,26 @@ war-rooms.
78
The user can also put the Cloud endpoint's full certificate chain in `claim.d/cloud_fullchain.pem` so that the agent
79
can trust the endpoint if necessary.
80
81
+## Using a proxy
82
+
83
+Claiming can be performed through a SOCKS5 proxy. To do this when calling the script directly supply the proxy
84
+endpoint as:
85
+
86
+```
87
+netdata-claim.sh -token=MYTOKEN1234567 -rooms=room1,room2 -proxy=socks5h://127.0.0.1:11081
88
+```
89
+
90
+When claiming via the `netdata` binary set the following options in the config:
91
+```
92
+[agent_cloud_link]
93
+ proxy = socks5://X.X.X.X:YYYY
94
+```
95
+Proceed to claim using the command-line syntax:
96
+```
97
+/usr/sbin/netdata -D -W "claim -token=MYTOKEN1234567 -rooms=room1,room2"
98
+```
99
+
100
+Please note - if you supply the proxy endpoint in the configuration then it will also be used to tunnel
101
+the agent cloud link as well.
102
+
103
[](<>)
claim/claim.c
+24
-2
@@ -2,6 +2,7 @@
2
3
#include "claim.h"
4
#include "../registry/registry_internals.h"
5
+#include "../aclk/aclk_common.h"
6
7
char *claiming_pending_arguments = NULL;
8
@@ -30,6 +31,7 @@ char *is_agent_claimed(void)
31
}
32
33
#define CLAIMING_COMMAND_LENGTH 16384
34
+#define CLAIMING_PROXY_LENGTH CLAIMING_COMMAND_LENGTH/4
35
36
extern struct registry registry;
37
@@ -46,12 +48,32 @@ void claim_agent(char *claiming_arguments)
48
char command_buffer[CLAIMING_COMMAND_LENGTH + 1];
49
FILE *fp;
50
51
+ char *cloud_base_hostname = NULL; // Initializers are over-written but prevent gcc complaining about clobbering.
52
+ char *cloud_base_port = NULL;
53
+ char *cloud_base_url = config_get(CONFIG_SECTION_CLOUD, "cloud base url", "https://netdata.cloud");
54
+ if( aclk_decode_base_url(cloud_base_url, &cloud_base_hostname, &cloud_base_port))
55
+ {
56
+ error("Configuration error - cannot decode \"cloud base url\"");
57
+ return;
58
+ }
59
+
60
+ const char *proxy_str;
61
+ ACLK_PROXY_TYPE proxy_type;
62
+ char proxy_flag[CLAIMING_PROXY_LENGTH] = "-noproxy";
63
+
64
+ proxy_str = aclk_lws_wss_get_proxy_setting(&proxy_type);
65
+
66
+ if(proxy_type == PROXY_TYPE_SOCKS5)
67
+ snprintf(proxy_flag, CLAIMING_PROXY_LENGTH, "-proxy=\"%s\"", proxy_str);
68
+
69
snprintfz(command_buffer,
70
CLAIMING_COMMAND_LENGTH,
51
- "exec netdata-claim.sh -hostname=%s -id=%s -url=%s %s",
71
+ "exec netdata-claim.sh %s -hostname=%s -id=%s -url=%s %s",
72
+
73
+ proxy_flag,
74
netdata_configured_hostname,
75
localhost->machine_guid,
54
- registry.cloud_base_url,
76
+ cloud_base_url,
77
claiming_arguments);
78
79
info("Executing agent claiming command 'netdata-claim.sh'");
claim/netdata-claim.sh.in
+108
-67
@@ -76,16 +76,16 @@
76
# Exit code: 15
77
78
if command -v curl >/dev/null 2>&1 ; then
79
- URLTOOL="curl"
79
+ URLTOOL="curl"
80
elif command -v wget >/dev/null 2>&1 ; then
81
- URLTOOL="wget"
81
+ URLTOOL="wget"
82
else
83
- echo >&2 "I need curl or wget to proceed, but neither is available on this system."
84
- exit 3
83
+ echo >&2 "I need curl or wget to proceed, but neither is available on this system."
84
+ exit 3
85
fi
86
if ! command -v openssl >/dev/null 2>&1 ; then
87
- echo >&2 "I need openssl to proceed, but neither is available on this system."
88
- exit 3
87
+ echo >&2 "I need openssl to proceed, but it is not available on this system."
88
+ exit 3
89
fi
90
91
@@ -101,36 +101,48 @@ ID="unknown"
101
ROOMS=""
102
HOSTNAME=$(hostname)
103
CLOUD_CERTIFICATE_FILE="${CLAIMING_DIR}/cloud_fullchain.pem"
104
+VERBOSE=0
105
+INSECURE=0
106
107
# get the MACHINE_GUID by default
108
if [ -r "${MACHINE_GUID_FILE}" ]; then
107
- ID="$(cat "${MACHINE_GUID_FILE}")"
109
+ ID="$(cat "${MACHINE_GUID_FILE}")"
110
fi
111
112
# get token from file
113
if [ -r "${CLAIMING_DIR}/token" ]; then
112
- TOKEN="$(cat "${CLAIMING_DIR}/token")"
114
+ TOKEN="$(cat "${CLAIMING_DIR}/token")"
115
fi
116
117
# get rooms from file
118
if [ -r "${CLAIMING_DIR}/rooms" ]; then
117
- ROOMS="$(cat "${CLAIMING_DIR}/rooms")"
119
+ ROOMS="$(cat "${CLAIMING_DIR}/rooms")"
120
fi
121
122
for arg in "$@"
123
do
122
- case $arg in
123
- -token=*) TOKEN=${arg:7} ;;
124
- -url=*) URL_BASE=${arg:5} ;;
125
- -id=*) ID=${arg:4} ;;
126
- -rooms=*) ROOMS=${arg:7} ;;
127
- -hostname=*) HOSTNAME=${arg:10} ;;
128
- *) echo >&2 "Unknown argument ${arg}"
129
- exit 1 ;;
130
- esac
131
- shift 1
124
+ case $arg in
125
+ -token=*) TOKEN=${arg:7} ;;
126
+ -url=*) URL_BASE=${arg:5} ;;
127
+ -id=*) ID=${arg:4} ;;
128
+ -rooms=*) ROOMS=${arg:7} ;;
129
+ -hostname=*) HOSTNAME=${arg:10} ;;
130
+ -verbose) VERBOSE=1 ;;
131
+ -insecure) INSECURE=1 ;;
132
+ -proxy=socks*) PROXY=${arg:7} ;;
133
+ -noproxy) NOPROXY=yes ;;
134
+ *) echo >&2 "Unknown argument ${arg}"
135
+ exit 1 ;;
136
+ esac
137
+ shift 1
138
done
139
140
+# if curl not installed give warning SOCKS can't be used
141
+if [[ "${URLTOOL}" != "curl" && "${PROXY:0:5}" = socks ]] ; then
142
+ echo >&2 "wget doesn't support SOCKS. Please install curl or disable SOCKS proxy."
143
+ exit 1
144
+fi
145
+
146
echo >&2 "Token: ****************"
147
echo >&2 "Base URL: $URL_BASE"
148
echo >&2 "Id: $ID"
@@ -139,34 +151,34 @@ echo >&2 "Hostname: $HOSTNAME"
151
152
# create the claiming directory for this user
153
if [ ! -d "${CLAIMING_DIR}" ] ; then
142
- mkdir -p "${CLAIMING_DIR}" && chmod 0770 "${CLAIMING_DIR}"
154
+ mkdir -p "${CLAIMING_DIR}" && chmod 0770 "${CLAIMING_DIR}"
155
# shellcheck disable=SC2181
144
- if [ $? -ne 0 ] ; then
145
- echo >&2 "Failed to create claiming working directory ${CLAIMING_DIR}"
146
- exit 2
147
- fi
156
+ if [ $? -ne 0 ] ; then
157
+ echo >&2 "Failed to create claiming working directory ${CLAIMING_DIR}"
158
+ exit 2
159
+ fi
160
fi
161
if [ ! -w "${CLAIMING_DIR}" ] ; then
150
- echo >&2 "No write permission in claiming working directory ${CLAIMING_DIR}"
151
- exit 2
162
+ echo >&2 "No write permission in claiming working directory ${CLAIMING_DIR}"
163
+ exit 2
164
fi
165
166
if [ ! -f "${CLAIMING_DIR}/private.pem" ] ; then
155
- echo >&2 "Generating private/public key for the first time."
156
- if ! openssl genrsa -out "${CLAIMING_DIR}/private.pem" 2048 ; then
157
- echo >&2 "Failed to generate private/public key pair."
158
- exit 2
159
- fi
167
+ echo >&2 "Generating private/public key for the first time."
168
+ if ! openssl genrsa -out "${CLAIMING_DIR}/private.pem" 2048 ; then
169
+ echo >&2 "Failed to generate private/public key pair."
170
+ exit 2
171
+ fi
172
fi
173
if [ ! -f "${CLAIMING_DIR}/public.pem" ] ; then
162
- echo >&2 "Extracting public key from private key."
163
- if ! openssl rsa -in "${CLAIMING_DIR}/private.pem" -outform PEM -pubout -out "${CLAIMING_DIR}/public.pem" ; then
164
- echo >&2 "Failed to extract public key."
165
- exit 2
166
- fi
174
+ echo >&2 "Extracting public key from private key."
175
+ if ! openssl rsa -in "${CLAIMING_DIR}/private.pem" -outform PEM -pubout -out "${CLAIMING_DIR}/public.pem" ; then
176
+ echo >&2 "Failed to extract public key."
177
+ exit 2
178
+ fi
179
fi
180
169
-TARGET_URL="${URL_BASE}/api/v1/spaces/nodes/${ID}"
181
+TARGET_URL="${URL_BASE%/}/api/v1/spaces/nodes/${ID}"
182
# shellcheck disable=SC2002
183
KEY=$(cat "${CLAIMING_DIR}/public.pem" | tr '\n' '!' | sed -e 's/!/\\n/g')
184
# shellcheck disable=SC2001
@@ -184,57 +196,86 @@ cat > "${CLAIMING_DIR}/tmpin.txt" <<EMBED_JSON
196
}
197
EMBED_JSON
198
199
+if [ "${VERBOSE}" == 1 ] ; then
200
+ echo "Request to server:"
201
+ cat "${CLAIMING_DIR}/tmpin.txt"
202
+fi
203
+
204
205
if [ "${URLTOOL}" = "curl" ] ; then
189
- URLCOMMAND="curl --connect-timeout 5 --retry 3 -s -i -X PUT -d \"@${CLAIMING_DIR}/tmpin.txt\""
206
+ URLCOMMAND="curl --connect-timeout 5 --retry 3 -s -i -X PUT -d \"@${CLAIMING_DIR}/tmpin.txt\""
207
+ if [ "${NOPROXY}" = "yes" ] ; then
208
+ URLCOMMAND="${URLCOMMAND} -x \"\""
209
+ elif [ -n "${PROXY}" ] ; then
210
+ URLCOMMAND="${URLCOMMAND} -x \"${PROXY}\""
211
+ fi
212
else
191
- URLCOMMAND="wget -T 15 -O - -q --save-headers --content-on-error=on --method=PUT \
192
- --body-file=\"${CLAIMING_DIR}/tmpin.txt\""
213
+ URLCOMMAND="wget -T 15 -O - -q --save-headers --content-on-error=on --method=PUT \
214
+ --body-file=\"${CLAIMING_DIR}/tmpin.txt\""
215
+ if [ "${NOPROXY}" = "yes" ] ; then
216
+ URLCOMMAND="${URLCOMMAND} --no-proxy"
217
+ fi
218
+fi
219
+
220
+if [ "${INSECURE}" == 1 ] ; then
221
+ if [ "${URLTOOL}" = "curl" ] ; then
222
+ URLCOMMAND="${URLCOMMAND} --insecure"
223
+ else
224
+ URLCOMMAND="${URLCOMMAND} --no-check-certificate"
225
+ fi
226
fi
227
228
if [ -r "${CLOUD_CERTIFICATE_FILE}" ] ; then
196
- if [ "${URLTOOL}" = "curl" ] ; then
197
- URLCOMMAND="${URLCOMMAND} --cacert \"${CLOUD_CERTIFICATE_FILE}\""
198
- else
199
- URLCOMMAND="${URLCOMMAND} --ca-certificate \"${CLOUD_CERTIFICATE_FILE}\""
200
- fi
229
+ if [ "${URLTOOL}" = "curl" ] ; then
230
+ URLCOMMAND="${URLCOMMAND} --cacert \"${CLOUD_CERTIFICATE_FILE}\""
231
+ else
232
+ URLCOMMAND="${URLCOMMAND} --ca-certificate \"${CLOUD_CERTIFICATE_FILE}\""
233
+ fi
234
fi
235
203
-eval "${URLCOMMAND} \"${TARGET_URL}\"" | tee "${CLAIMING_DIR}/tmpout.txt"
236
+if [ "${VERBOSE}" == 1 ]; then
237
+ echo "${URLCOMMAND} \"${TARGET_URL}\""
238
+fi
239
+eval "${URLCOMMAND} \"${TARGET_URL}\"" >"${CLAIMING_DIR}/tmpout.txt"
240
URLCOMMAND_EXIT_CODE=$?
241
if [ "${URLTOOL}" = "wget" ] && [ "${URLCOMMAND_EXIT_CODE}" -eq 8 ] ; then
242
# We consider the server issuing an error response a successful attempt at communicating
207
- URLCOMMAND_EXIT_CODE=0
243
+ URLCOMMAND_EXIT_CODE=0
244
fi
245
246
rm -f "${CLAIMING_DIR}/tmpin.txt"
247
248
# Check if URLCOMMAND connected and received reply
249
if [ "${URLCOMMAND_EXIT_CODE}" -ne 0 ] ; then
214
- echo >&2 "Failed to connect to ${URL_BASE}"
215
- rm -f "${CLAIMING_DIR}/tmpout.txt"
216
- exit 4
250
+ echo >&2 "Failed to connect to ${URL_BASE}, return code ${URLCOMMAND_EXIT_CODE}"
251
+ rm -f "${CLAIMING_DIR}/tmpout.txt"
252
+ exit 4
253
+fi
254
+
255
+if [ "${VERBOSE}" == 1 ] ; then
256
+ echo "Response from server:"
257
+ cat "${CLAIMING_DIR}/tmpout.txt"
258
fi
259
260
HTTP_STATUS_CODE=$(grep "HTTP" "${CLAIMING_DIR}/tmpout.txt" | awk -F " " '{print $2}')
261
if [ "${HTTP_STATUS_CODE}" -ne 204 ] ; then
221
- ERROR_MESSAGE=$(grep "\"errorMsgKey\":" "${CLAIMING_DIR}/tmpout.txt" | awk -F "errorMsgKey\":\"" '{print $2}' | awk -F "\"" '{print $1}')
222
- case ${ERROR_MESSAGE} in
223
- "ErrInvalidNodeID") EXIT_CODE=6 ;;
224
- "ErrInvalidNodeName") EXIT_CODE=7 ;;
225
- "ErrInvalidRoomID") EXIT_CODE=8 ;;
226
- "ErrInvalidPublicKey") EXIT_CODE=9 ;;
227
- "ErrForbidden") EXIT_CODE=10 ;;
228
- "ErrAlreadyClaimed") EXIT_CODE=11 ;;
229
- "ErrProcessingClaim") EXIT_CODE=12 ;;
230
- "ErrInternalServerError") EXIT_CODE=13 ;;
231
- "ErrGatewayTimeout") EXIT_CODE=14 ;;
232
- "ErrServiceUnavailable") EXIT_CODE=15 ;;
233
- *) EXIT_CODE=5 ;;
234
- esac
235
- echo >&2 "Failed to claim node."
236
- rm -f "${CLAIMING_DIR}/tmpout.txt"
237
- exit $EXIT_CODE
262
+ ERROR_MESSAGE=$(grep "\"errorMsgKey\":" "${CLAIMING_DIR}/tmpout.txt" | awk -F "errorMsgKey\":\"" '{print $2}' | awk -F "\"" '{print $1}')
263
+ case ${ERROR_MESSAGE} in
264
+ "ErrInvalidNodeID") EXIT_CODE=6 ;;
265
+ "ErrInvalidNodeName") EXIT_CODE=7 ;;
266
+ "ErrInvalidRoomID") EXIT_CODE=8 ;;
267
+ "ErrInvalidPublicKey") EXIT_CODE=9 ;;
268
+ "ErrForbidden") EXIT_CODE=10 ;;
269
+ "ErrAlreadyClaimed") EXIT_CODE=11 ;;
270
+ "ErrProcessingClaim") EXIT_CODE=12 ;;
271
+ "ErrInternalServerError") EXIT_CODE=13 ;;
272
+ "ErrGatewayTimeout") EXIT_CODE=14 ;;
273
+ "ErrServiceUnavailable") EXIT_CODE=15 ;;
274
+ *) EXIT_CODE=5 ;;
275
+ esac
276
+ echo >&2 "Failed to claim node."
277
+ rm -f "${CLAIMING_DIR}/tmpout.txt"
278
+ exit $EXIT_CODE
279
fi
280
281
rm -f "${CLAIMING_DIR}/tmpout.txt"