@cryptotaxi247 / netdata-1 / commits / 3076cfe5d

Url parser refactoring (#6247)

* URL_parser_review comments 1 * URL_parser_review restoring web_client.c * URL_parser_review restoring url.h * URL_parser_review restoring web_client.h * URL_parser_review restoring inlined.h * URL_parser_review restoring various * URL_parser_review commenting! * URL_parser_review last checks! * URL_parser_review registry! * URL_parser_review codacy errors! * URL_parser_review codacy errors 2! * URL_parser_review end of request! * URL_parser_review * URL_parser_review format fix * URL_parser_review restoring * URL_parser_review stopped at 5! * URL_parser_review formatting! * URL_parser_review: Started the map of the query string when it is necessary * URL_parser_review: With these adjusts in the URL library we are now able to parser all the escape characters! * URL_parser_review: code review Fixes problems and format asked by coworkers! * URL_parser_review: adjust script The script was not 100% according the shellcheck specifications, no less important it was a direct script instead a .in file * sslstream: Rebase 2 It was necessary to change a function due the UTF-8 * sslstream: Fixing 6426 We had a cast error introduced by other PR, so I am fixing here * URL_parser_review Change .gitignore to avoid considering a script file.

thiagoftsm committed Jul 25, 2019 at 12:30 UTC 3076cfe5d455b8007e4f90776e9ea3d05faf1a7e
9 files changed +711 -79
.gitignore
+1
@@ -166,6 +166,7 @@ callgrind.out.*
166 gmon.out
167 gmon.txt
168 sitespeed-result/
169 +tests/urls/request.sh
170
171 # tests and temp files
172 python.d/python-modules-installer.sh
libnetdata/url/url.c
+194 -8
@@ -43,8 +43,16 @@ char *url_encode(char *str) {
43 return pbuf;
44 }
45
46 -/* Returns a url-decoded version of str */
47 -/* IMPORTANT: be sure to free() the returned string after use */
46 +/**
47 + * URL Decode
48 + *
49 + * Returns a url-decoded version of str
50 + * IMPORTANT: be sure to free() the returned string after use
51 + *
52 + * @param str the string that will be decode
53 + *
54 + * @return a pointer for the url decoded.
55 + */
56 char *url_decode(char *str) {
57 size_t size = strlen(str) + 1;
58
@@ -52,14 +60,30 @@ char *url_decode(char *str) {
60 return url_decode_r(buf, str, size);
61 }
62
55 -//decode %XX character or return 0 if cannot
63 +/**
64 + * Percentage escape decode
65 + *
66 + * Decode %XX character or return 0 if cannot
67 + *
68 + * @param s the string to decode
69 + *
70 + * @return The character decoded on success and 0 otherwise
71 + */
72 char url_percent_escape_decode(char *s) {
73 if(likely(s[1] && s[2]))
74 return from_hex(s[1]) << 4 | from_hex(s[2]);
75 return 0;
76 }
77
62 -//this (utf8 string related) should be moved in separate file in future
78 +/**
79 + * Get byte length
80 + *
81 + * This (utf8 string related) should be moved in separate file in future
82 + *
83 + * @param c is the utf8 character
84 + * *
85 + * @return It reurns the length of the specific character.
86 + */
87 char url_utf8_get_byte_length(char c) {
88 if(!IS_UTF8_BYTE(c))
89 return 1;
@@ -77,8 +101,17 @@ char url_utf8_get_byte_length(char c) {
101 return length;
102 }
103
80 -//decode % encoded UTF-8 characters and copy them to *d
81 -//return count of bytes written to *d
104 +/**
105 + * Decode Multibyte UTF8
106 + *
107 + * Decode % encoded UTF-8 characters and copy them to *d
108 + *
109 + * @param s first address
110 + * @param d
111 + * @param d_end last address
112 + *
113 + * @return count of bytes written to *d
114 + */
115 char url_decode_multibyte_utf8(char *s, char *d, char *d_end) {
116 char first_byte = url_percent_escape_decode(s);
117
@@ -122,7 +155,6 @@ char url_decode_multibyte_utf8(char *s, char *d, char *d_end) {
155 * Markus Kuhn <http://www.cl.cam.ac.uk/~mgk25/> -- 2005-03-30
156 * License: http://www.cl.cam.ac.uk/~mgk25/short-license.html
157 */
125 -
158 unsigned char *utf8_check(unsigned char *s)
159 {
160 while (*s)
@@ -208,7 +240,7 @@ char *url_decode_r(char *to, char *url, size_t size) {
240
241 *d = '\0';
242
211 - if(unlikely( utf8_check(to) )) //NULL means sucess here
243 + if(unlikely( utf8_check((unsigned char *)to) )) //NULL means sucess here
244 return NULL;
245
246 return to;
@@ -217,3 +249,157 @@ fail_cleanup:
249 *d = '\0';
250 return NULL;
251 }
252 +
253 +/**
254 + * Is request complete?
255 + *
256 + * Check whether the request is complete.
257 + * This function cannot check all the requests METHODS, for example, case you are working with POST, it will fail.
258 + *
259 + * @param begin is the first character of the sequence to analyse.
260 + * @param end is the last character of the sequence
261 + * @param length is the length of the total of bytes read, it is not the difference between end and begin.
262 + *
263 + * @return It returns 1 when the request is complete and 0 otherwise.
264 + */
265 +inline int url_is_request_complete(char *begin, char *end, size_t length) {
266 +
267 + if ( begin == end) {
268 + //Message cannot be complete when first and last address are the same
269 + return 0;
270 + }
271 +
272 + //This math to verify the last is valid, because we are discarding the POST
273 + if (length > 4) {
274 + begin = end - 4;
275 + }
276 +
277 + return (strstr(begin, "\r\n\r\n"))?1:0;
278 +}
279 +
280 +/**
281 + * Find protocol
282 + *
283 + * Search for the string ' HTTP/' in the message given.
284 + *
285 + * @param s is the start of the user request.
286 + * @return
287 + */
288 +inline char *url_find_protocol(char *s) {
289 + while(*s) {
290 + // find the next space
291 + while (*s && *s != ' ') s++;
292 +
293 + // is it SPACE + "HTTP/" ?
294 + if(*s && !strncmp(s, " HTTP/", 6)) break;
295 + else s++;
296 + }
297 +
298 + return s;
299 +}
300 +
301 +/**
302 + * Map query string
303 + *
304 + * Map the query string fields that will be decoded.
305 + * This functions must be called after to check the presence of query strings,
306 + * here we are assuming that you already tested this.
307 + *
308 + * @param out the pointer to pointers that will be used to map
309 + * @param url the input url that we are decoding.
310 + *
311 + * @return It returns the number of total variables in the query string.
312 + */
313 +int url_map_query_string(char **out, char *url) {
314 + (void)out;
315 + (void)url;
316 + int count = 0;
317 +
318 + //First we try to parse considering that there was not URL encode process
319 + char *moveme = url;
320 + char *ptr;
321 +
322 + //We always we have at least one here, so I can set this.
323 + out[count++] = moveme;
324 + while(moveme) {
325 + ptr = strchr((moveme+1), '&');
326 + if(ptr) {
327 + out[count++] = ptr;
328 + }
329 +
330 + moveme = ptr;
331 + }
332 +
333 + //I could not find any '&', so I am assuming now it is like '%26'
334 + if (count == 1) {
335 + moveme = url;
336 + while(moveme) {
337 + ptr = strchr((moveme+1), '%');
338 + if(ptr) {
339 + char *test = (ptr+1);
340 + if (!strncmp(test, "3f", 2) || !strncmp(test, "3F", 2)) {
341 + out[count++] = ptr;
342 + }
343 + }
344 + moveme = ptr;
345 + }
346 + }
347 +
348 + return count;
349 +}
350 +
351 +/**
352 + * Parse query string
353 + *
354 + * Parse the query string mapped and store it inside output.
355 + *
356 + * @param output is a vector where I will store the string.
357 + * @param max is the maximum length of the output
358 + * @param map the map done by the function url_map_query_string.
359 + * @param total the total number of variables inside map
360 + *
361 + * @return It returns 0 on success and -1 otherwise
362 + */
363 +int url_parse_query_string(char *output, size_t max, char **map, int total) {
364 + if(!total) {
365 + return 0;
366 + }
367 +
368 + int counter, next;
369 + size_t length;
370 + char *end;
371 + char *begin = map[0];
372 + char save;
373 + size_t copied = 0;
374 + for(counter = 0, next=1 ; next <= total ; ++counter, ++next) {
375 + if (next != total) {
376 + end = map[next];
377 + length = (size_t) (end - begin);
378 + save = *end;
379 + *end = 0x00;
380 + } else {
381 + length = strlen(begin);
382 + end = NULL;
383 + }
384 + length++;
385 +
386 + if (length > (max - copied)) {
387 + error("Parsing query string: we cannot parse a query string so big");
388 + break;
389 + }
390 +
391 + if(!url_decode_r(output, begin, length)) {
392 + return -1;
393 + }
394 + length = strlen(output);
395 + copied += length;
396 + output += length;
397 +
398 + begin = end;
399 + if (begin) {
400 + *begin = save;
401 + }
402 + }
403 +
404 + return 0;
405 +}
libnetdata/url/url.h
+7
@@ -25,4 +25,11 @@ extern char *url_decode(char *str);
25
26 extern char *url_decode_r(char *to, char *url, size_t size);
27
28 +#define WEB_FIELDS_MAX 400
29 +extern int url_map_query_string(char **out, char *url);
30 +extern int url_parse_query_string(char *output, size_t max, char **map, int total);
31 +
32 +extern int url_is_request_complete(char *begin,char *end,size_t length);
33 +extern char *url_find_protocol(char *s);
34 +
35 #endif /* NETDATA_URL_H */
tests/Makefile.am
+3
@@ -5,6 +5,7 @@ MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
6 CLEANFILES = \
7 health_mgmtapi/health-cmdapi-test.sh \
8 + urls/request.sh \
9 $(NULL)
10
11 include $(top_srcdir)/build/subst.inc
@@ -22,10 +23,12 @@ dist_noinst_DATA = \
23 node.d/fronius.process.spec.js \
24 node.d/fronius.validation.spec.js \
25 health_mgmtapi/health-cmdapi-test.sh.in \
26 + urls/request.sh.in \
27 $(NULL)
28
29 dist_plugins_SCRIPTS = \
30 health_mgmtapi/health-cmdapi-test.sh \
31 + urls/request.sh \
32 $(NULL)
33
34 dist_noinst_SCRIPTS = \
tests/urls/request.sh.in new
+303
@@ -0,0 +1,303 @@
1 +#!/bin/bash
2 +# SPDX-License-Identifier: GPL-3.0-or-later
3 +
4 +################################################################################################
5 +#### ####
6 +#### GLOBAL VARIABLES ####
7 +#### ####
8 +################################################################################################
9 +
10 +# The current time
11 +CT=$(date +'%s')
12 +
13 +# The previous time
14 +PT=$((CT - 30))
15 +
16 +# The output directory where we will store the results and error
17 +OUTDIR="tests"
18 +OUTEDIR="encoded_tests"
19 +OUTOPTDIR="options"
20 +ERRDIR="etests"
21 +
22 +################################################################################################
23 +#### ####
24 +#### FUNCTIONS ####
25 +#### ####
26 +################################################################################################
27 +
28 +# Print error message and close script
29 +netdata_print_error(){
30 + echo "Closing due error \"$1\" code \"$2\""
31 + exit 1
32 +}
33 +
34 +# Print the header message of the function
35 +netdata_print_header() {
36 + echo "$1"
37 +}
38 +
39 +# Create the main directory where the results will be stored
40 +netdata_create_directory() {
41 + netdata_print_header "Creating directory $1"
42 + if [ ! -d "$1" ]; then
43 + mkdir "$1"
44 + TEST=$?
45 + if [ $TEST -ne 0 ]; then
46 + netdata_print_error "Cannot create directory $?"
47 + fi
48 + else
49 + echo "Working with directory $OUTDIR"
50 + fi
51 +}
52 +
53 +#Check whether download did not have problem
54 +netdata_test_download(){
55 + grep "HTTP/1.1 200 OK" "$1" 2>/dev/null 1>/dev/null
56 + TEST=$?
57 + if [ $TEST -ne 0 ]; then
58 + netdata_print_error "Cannot do download of the page $2" $?
59 + exit 1
60 + fi
61 +}
62 +
63 +#Check whether download had a problem
64 +netdata_error_test(){
65 + grep "HTTP/1.1 200 OK" "$1" 2>/dev/null 1>/dev/null
66 + TEST=$?
67 + if [ $TEST -eq 0 ]; then
68 + netdata_print_error "The page $2 did not answer with an error" $?
69 + exit 1
70 + fi
71 +}
72 +
73 +
74 +# Download information from Netdata
75 +netdata_download_various() {
76 + netdata_print_header "Getting $2"
77 + curl -v -k --create-dirs -o "$OUTDIR/$3.out" "$1/$2" 2> "$OUTDIR/$3.err"
78 + netdata_test_download "$OUTDIR/$3.err" "$1/$2"
79 +}
80 +
81 +netdata_download_various_with_options() {
82 + netdata_print_header "Getting options for $2"
83 + curl -X OPTIONS -v -k --create-dirs -o "$OUTOPTDIR/$3.out" "$1/$2" 2> "$OUTOPTDIR/$3.err"
84 + netdata_test_download "$OUTOPTDIR/$3.err" "$1/$2"
85 +}
86 +
87 +# Download information from Netdata
88 +netdata_wrong_request_various() {
89 + netdata_print_header "Getting $2"
90 + curl -v -k --create-dirs -o "$ERRDIR/$3.out" "$1/$2" 2> "$ERRDIR/$3.err"
91 + netdata_error_test "$ERRDIR/$3.err" "$1/$2"
92 +}
93 +
94 +# Download charts from Netdata
95 +netdata_download_charts() {
96 + curl -v -k --create-dirs -o "$OUTDIR/charts.out" "$1/$2" 2> "$OUTDIR/charts.err"
97 + netdata_test_download "$OUTDIR/charts.err" "$1/$2"
98 +
99 + #Rewrite the next
100 + grep -w "id" tests/charts.out| cut -d: -f2 | grep "\"," | sed s/,//g | sort
101 +}
102 +
103 +#Test options for a specific chart
104 +netdata_download_chart() {
105 + SEPARATOR="&"
106 + EQUAL="="
107 + OUTD=$OUTDIR
108 + ENCODED=" "
109 + for I in $(seq 0 1); do
110 + if [ "$I" -eq "1" ] ; then
111 + SEPARATOR="%26"
112 + EQUAL="%3D"
113 + OUTD=$OUTEDIR
114 + ENCODED="encoded"
115 + fi
116 +
117 + NAME=${3//\"/}
118 + netdata_print_header "Getting data for $NAME using $4 $ENCODED"
119 +
120 + LDIR=$OUTD"/"$4
121 +
122 + LURL="$1/$2$EQUAL$NAME"
123 +
124 + NAME=$NAME"_$4"
125 +
126 + curl -v -k --create-dirs -o "$LDIR/$NAME.out" "$LURL" 2> "$LDIR/$NAME.err"
127 + netdata_test_download "$LDIR/$NAME.err" "$LURL"
128 +
129 + UFILES=( "points" "before" "after" )
130 + COUNTER=0
131 + for OPT in "points=100" "before=$PT" "after=$CT" ;
132 + do
133 + LURL="$LURL$SEPARATOR$OPT"
134 + LFILE=$NAME"_${UFILES[$COUNTER]}";
135 +
136 + curl -v -k --create-dirs -o "$LDIR/$LFILE.out" "$LURL" 2> "$LDIR/$LFILE.err"
137 + netdata_test_download "$LDIR/$LFILE.err" "$LURL"
138 +
139 + COUNTER=$((COUNTER + 1))
140 + done
141 +
142 + LURL="$LURL&group$EQUAL"
143 + for OPT in "min" "max" "sum" "median" "stddev" "cv" "ses" "des" "incremental_sum" "average";
144 + do
145 + TURL=$LURL$OPT
146 + TFILE=$NAME"_$OPT";
147 + curl -v -k --create-dirs -o "$LDIR/$TFILE.out" "$TURL" 2> "$LDIR/$TFILE.err"
148 + netdata_test_download "$LDIR/$TFILE.err" "$TURL"
149 + for MORE in "jsonp" "json" "ssv" "csv" "datatable" "datasource" "tsv" "ssvcomma" "html" "array";
150 + do
151 + TURL=$TURL"&format="$MORE
152 + TFILE=$NAME"_$OPT""_$MORE";
153 + curl -v -k --create-dirs -o "$LDIR/$TFILE.out" "$TURL" 2> "$LDIR/$TFILE.err"
154 + netdata_test_download "$LDIR/$TFILE.err" "$TURL"
155 + done
156 + done
157 +
158 + LURL="$LURL$OPT&gtime=60"
159 + NFILE=$NAME"_gtime"
160 + curl -v -k --create-dirs -o "$LDIR/$NFILE.out" "$TURL" 2> "$LDIR/$NFILE.err"
161 + netdata_test_download "$LDIR/$NFILE.err" "$LURL"
162 +
163 + LURL="$LURL$OPT&options=percentage"
164 + NFILE=$NAME"_percentage"
165 + curl -v -k --create-dirs -o "$LDIR/$NFILE.out" "$TURL" 2> "$LDIR/$NFILE.err"
166 + netdata_test_download "$LDIR/$NFILE.err" "$LURL"
167 +
168 + LURL="$LURL$OPT&dimensions=system%7Cnice"
169 + NFILE=$NAME"_dimension"
170 + curl -v -k --create-dirs -o "$LDIR/$NFILE.out" "$TURL" 2> "$LDIR/$NFILE.err"
171 + netdata_test_download "$LDIR/$NFILE.err" "$LURL"
172 +
173 + LURL="$LURL$OPT&label=testing"
174 + NFILE=$NAME"_label"
175 + curl -v -k --create-dirs -o "$LDIR/$NFILE.out" "$TURL" 2> "$LDIR/$NFILE.err"
176 + netdata_test_download "$LDIR/$NFILE.err" "$LURL"
177 + done
178 +}
179 +
180 +# Download information from Netdata
181 +netdata_download_allmetrics() {
182 + netdata_print_header "Getting All metrics"
183 + LURL="$1/api/v1/allmetrics?format="
184 + for FMT in "shell" "prometheus" "prometheus_all_hosts" "json" ;
185 + do
186 + TURL=$LURL$FMT
187 + for OPT in "yes" "no";
188 + do
189 + if [ "$FMT" == "prometheus" ]; then
190 + TURL="$TURL&help=$OPT&types=$OPT&timestamps=$OPT"
191 + fi
192 + TURL="$TURL&names=$OPT&oldunits=$OPT&hideunits=$OPT&prefix=ND"
193 +
194 + NAME="allmetrics_$FMT"
195 + echo "$OUTDIR/$2/$NAME.out"
196 + curl -v -k --create-dirs -o "$OUTDIR/$2/$NAME.out" "$TURL" 2> "$OUTDIR/$2/$NAME.err"
197 + netdata_test_download "$OUTDIR/$2/$NAME.err" "$TURL"
198 + done
199 + done
200 +}
201 +
202 +
203 +################################################################################################
204 +#### ####
205 +#### MAIN ROUTINE ####
206 +#### ####
207 +################################################################################################
208 +MURL="http://127.0.0.1:19999"
209 +
210 +netdata_create_directory $OUTDIR
211 +netdata_create_directory $OUTEDIR
212 +netdata_create_directory $OUTOPTDIR
213 +netdata_create_directory $ERRDIR
214 +
215 +wget --execute="robots = off" --mirror --convert-links --no-parent http://127.0.0.1:19999
216 +TEST=$?
217 +if [ $TEST -ne "0" ] ; then
218 + echo "Cannot connect to Netdata"
219 + exit 1
220 +fi
221 +
222 +netdata_download_various $MURL "netdata.conf" "netdata.conf"
223 +
224 +netdata_download_various_with_options $MURL "netdata.conf" "netdata.conf"
225 +
226 +netdata_wrong_request_various $MURL "api/v15/info?this%20could%20not%20be%20here" "err_version"
227 +
228 +netdata_wrong_request_various $MURL "api/v1/\(*@&$\!$%%5E\)\!$*%&\)\!$*%%5E*\!%5E%\!%5E$%\!%5E%\(\!*%5E*%5E%\(*@&$%5E%\(\!%5E#*&\!^#$*&\!^%\)@\($%^\)\!*&^\(\!*&^#$&#$\)\!$%^\)\!$*%&\)#$\!^#*$^\!\(*#^#\)\!%^\!\)$*%&\!\(*&$\!^#$*&^\!*#^$\!*^\)%\(\!*&$%\)\(\!&#$\!^*#&$^\!*^%\)\!$%\)\!\(&#$\!^#*&^$" "err_version2"
229 +
230 +netdata_download_various $MURL "api/v1/info" "info"
231 +netdata_download_various_with_options $MURL "api/v1/info" "info"
232 +netdata_download_various $MURL "api/v1/info?this%20could%20not%20be%20here" "err_info"
233 +
234 +netdata_print_header "Getting all the netdata charts"
235 +CHARTS=$( netdata_download_charts "http://127.0.0.1:19999" "api/v1/charts" )
236 +WCHARTS=$( netdata_download_charts "http://127.0.0.1:19999" "api/v1/charts?this%20could%20not%20be%20here" )
237 +WCHARTS2=$( netdata_download_charts "http://127.0.0.1:19999" "api/v1/charts%3fthis%20could%20not%20be%20here" )
238 +
239 +if [ ${#CHARTS[@]} -ne ${#WCHARTS[@]} ]; then
240 + echo "The number of charts does not match with division not encoded.";
241 + exit 2;
242 +elif [ ${#CHARTS[@]} -ne ${#WCHARTS2[@]} ]; then
243 + echo "The number of charts does not match when everything is encoded";
244 + exit 3;
245 +fi
246 +
247 +netdata_wrong_request_various $MURL "api/v1/chart" "err_chart_without_chart"
248 +netdata_wrong_request_various $MURL "api/v1/chart?_=234231424242" "err_chart_arg"
249 +
250 +netdata_download_various $MURL "api/v1/chart?chart=cpu.cpu0_interrupts&_=234231424242" "chart_cpu_with_more_args"
251 +netdata_download_various_with_options $MURL "api/v1/chart?chart=cpu.cpu0_interrupts&_=234231424242" "chart_cpu_with_more_args"
252 +
253 +netdata_download_various $MURL "api/v1/chart%3Fchart=cpu.cpu0_interrupts&_=234231424242" "chart_cpu_with_more_args_encoded"
254 +netdata_download_various_with_options $MURL "api/v1/chart%3Fchart=cpu.cpu0_interrupts&_=234231424242" "chart_cpu_with_more_args_encoded"
255 +netdata_download_various $MURL "api/v1/chart%3Fchart=cpu.cpu0_interrupts%26_=234231424242" "chart_cpu_with_more_args_encoded2"
256 +netdata_download_various $MURL "api/v1/chart%3Fchart%3Dcpu.cpu0_interrupts%26_%3D234231424242" "chart_cpu_with_more_args_encoded3"
257 +
258 +netdata_create_directory "$OUTDIR/chart"
259 +for I in $CHARTS ; do
260 + NAME=${I//\"/}
261 + netdata_download_various $MURL "api/v1/chart?chart=$NAME" "chart/$NAME"
262 +done
263 +
264 +netdata_wrong_request_various $MURL "api/v1/alarm_variables" "err_alarm_variables_without_chart"
265 +netdata_wrong_request_various $MURL "api/v1/alarm_variables?_=234231424242" "err_alarm_variables_arg"
266 +netdata_download_various $MURL "api/v1/alarm_variables?chart=cpu.cpu0_interrupts&_=234231424242" "alarm_cpu_with_more_args"
267 +
268 +netdata_create_directory "$OUTDIR/alarm_variables"
269 +for I in $CHARTS ; do
270 + NAME=${I//\"/}
271 + netdata_download_various $MURL "api/v1/alarm_variables?chart=$NAME" "alarm_variables/$NAME"
272 +done
273 +
274 +netdata_create_directory "$OUTDIR/badge"
275 +netdata_create_directory "$OUTEDIR/badge"
276 +for I in $CHARTS ; do
277 + netdata_download_chart $MURL "api/v1/badge.svg?chart" "$I" "badge"
278 +done
279 +
280 +netdata_create_directory "$OUTDIR/allmetrics"
281 +netdata_download_allmetrics $MURL "allmetrics"
282 +
283 +netdata_download_various $MURL "api/v1/alarms?all" "alarms_all"
284 +netdata_download_various $MURL "api/v1/alarms?active" "alarms_active"
285 +netdata_download_various $MURL "api/v1/alarms" "alarms_nothing"
286 +
287 +netdata_download_various $MURL "api/v1/alarm_log?after" "alarm_without"
288 +netdata_download_various $MURL "api/v1/alarm_log" "alarm_nothing"
289 +netdata_download_various $MURL "api/v1/alarm_log?after&_=$PT" "alarm_log"
290 +
291 +netdata_create_directory "$OUTDIR/data"
292 +netdata_create_directory "$OUTEDIR/data"
293 +for I in $CHARTS ; do
294 + netdata_download_chart $MURL "api/v1/data?chart" "$I" "data"
295 + break;
296 +done
297 +
298 +#http://arch-esxi:19999/api/v1/(*@&$!$%%5E)!$*%&)!$*%%5E*!%5E%!%5E$%!%5E%(!*%5E*%5E%(*@&$%5E%(!%5E#*&!^#$*&!^%)@($%^)!*&^(!*&^#$&#$)!$%^)!$*%&)#$!^#*$^!(*#^#)!%^!)$*%&!(*&$!^#$*&^!*#^$!*^)%(!*&$%)(!&#$!^*#&$^!*^%)!$%)!(&#$!^#*&^$
299 +
300 +WHITE='\033[0;37m'
301 +echo -e "${WHITE}ALL the URLS got 200 as answer!"
302 +
303 +exit 0
web/api/health/health_cmdapi.c
+1
@@ -179,6 +179,7 @@ int web_client_api_request_v1_mgmt_health(RRDHOST *host, struct web_client *w, c
179 silencer = health_silencers_addparam(silencer, key, value);
180 }
181 }
182 +
183 if (likely(silencer)) {
184 health_silencers_add(silencer);
185 buffer_strcat(wb, HEALTH_CMDAPI_MSG_ADDED);
web/api/web_api_v1.c
+7 -7
@@ -797,23 +797,23 @@ inline int web_client_api_request_v1(RRDHOST *host, struct web_client *w, char *
797 }
798
799 // get the command
800 - char *tok = mystrsep(&url, "?");
801 - if(tok && *tok) {
802 - debug(D_WEB_CLIENT, "%llu: Searching for API v1 command '%s'.", w->id, tok);
803 - uint32_t hash = simple_hash(tok);
800 + if(url) {
801 + debug(D_WEB_CLIENT, "%llu: Searching for API v1 command '%s'.", w->id, url);
802 + uint32_t hash = simple_hash(url);
803
804 for(i = 0; api_commands[i].command ;i++) {
806 - if(unlikely(hash == api_commands[i].hash && !strcmp(tok, api_commands[i].command))) {
805 + if(unlikely(hash == api_commands[i].hash && !strcmp(url, api_commands[i].command))) {
806 if(unlikely(api_commands[i].acl != WEB_CLIENT_ACL_NOCHECK) && !(w->acl & api_commands[i].acl))
807 return web_client_permission_denied(w);
808
810 - return api_commands[i].callback(host, w, url);
809 + //return api_commands[i].callback(host, w, url);
810 + return api_commands[i].callback(host, w, (w->decoded_query_string + 1));
811 }
812 }
813
814 buffer_flush(w->response.data);
815 buffer_strcat(w->response.data, "Unsupported v1 API command: ");
816 - buffer_strcat_htmlescape(w->response.data, tok);
816 + buffer_strcat_htmlescape(w->response.data, url);
817 return 404;
818 }
819 else {
web/server/web_client.c
+178 -64
@@ -772,7 +772,6 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
772 // terminate the value
773 *ve = '\0';
774
775 - // fprintf(stderr, "HEADER: '%s' = '%s'\n", s, v);
775 uint32_t hash = simple_uhash(s);
776
777 if(hash == hash_origin && !strcasecmp(s, "Origin"))
@@ -812,66 +811,31 @@ static inline char *http_header_parse(struct web_client *w, char *s, int parse_u
811 return ve;
812 }
813
815 -// http_request_validate()
816 -// returns:
817 -// = 0 : all good, process the request
818 -// > 0 : request is not supported
819 -// < 0 : request is incomplete - wait for more data
820 -
821 -typedef enum {
822 - HTTP_VALIDATION_OK,
823 - HTTP_VALIDATION_NOT_SUPPORTED,
824 - HTTP_VALIDATION_MALFORMED_URL,
825 -#ifdef ENABLE_HTTPS
826 - HTTP_VALIDATION_INCOMPLETE,
827 - HTTP_VALIDATION_REDIRECT
828 -#else
829 - HTTP_VALIDATION_INCOMPLETE
830 -#endif
831 -} HTTP_VALIDATION;
832 -
833 -static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
834 - char *s = (char *)buffer_tostring(w->response.data), *encoded_url = NULL;
835 -
836 - size_t last_pos = w->header_parse_last_size;
837 - if(last_pos > 4) last_pos -= 4; // allow searching for \r\n\r\n
838 - else last_pos = 0;
839 -
840 - w->header_parse_tries++;
841 - w->header_parse_last_size = buffer_strlen(w->response.data);
842 -
843 - if(w->header_parse_tries > 1) {
844 - if(w->header_parse_last_size < last_pos)
845 - last_pos = 0;
846 -
847 - if(strstr(&s[last_pos], "\r\n\r\n") == NULL) {
848 - if(w->header_parse_tries > 10) {
849 - info("Disabling slow client after %zu attempts to read the request (%zu bytes received)", w->header_parse_tries, buffer_strlen(w->response.data));
850 - w->header_parse_tries = 0;
851 - w->header_parse_last_size = 0;
852 - web_client_disable_wait_receive(w);
853 - return HTTP_VALIDATION_NOT_SUPPORTED;
854 - }
855 -
856 - return HTTP_VALIDATION_INCOMPLETE;
857 - }
858 - }
859 -
814 +/**
815 + * Valid Method
816 + *
817 + * Netdata accepts only three methods, including one of these three(STREAM) is an internal method.
818 + *
819 + * @param w is the structure with the client request
820 + * @param s is the start string to parse
821 + *
822 + * @return it returns the next address to parse case the method is valid and NULL otherwise.
823 + */
824 +static inline char *web_client_valid_method(struct web_client *w, char *s) {
825 // is is a valid request?
826 if(!strncmp(s, "GET ", 4)) {
862 - encoded_url = s = &s[4];
827 + s = &s[4];
828 w->mode = WEB_CLIENT_MODE_NORMAL;
829 }
830 else if(!strncmp(s, "OPTIONS ", 8)) {
866 - encoded_url = s = &s[8];
831 + s = &s[8];
832 w->mode = WEB_CLIENT_MODE_OPTIONS;
833 }
834 else if(!strncmp(s, "STREAM ", 7)) {
835 + s = &s[7];
836 +
837 #ifdef ENABLE_HTTPS
838 if ( (w->ssl.flags) && (netdata_use_ssl_on_stream & NETDATA_SSL_FORCE)){
872 - w->header_parse_tries = 0;
873 - w->header_parse_last_size = 0;
874 - web_client_disable_wait_receive(w);
839 char hostname[256];
840 char *copyme = strstr(s,"hostname=");
841 if ( copyme ){
@@ -892,29 +856,150 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
856 hostname[13] = 0x00;
857 }
858 error("The server is configured to always use encrypt connection, please enable the SSL on slave with hostname '%s'.",hostname);
895 - return HTTP_VALIDATION_NOT_SUPPORTED;
859 + s = NULL;
860 }
861 #endif
862
899 - encoded_url = s = &s[7];
863 w->mode = WEB_CLIENT_MODE_STREAM;
864 }
865 else {
866 + s = NULL;
867 + }
868 +
869 + return s;
870 +}
871 +
872 +/**
873 + * Set Path Query
874 + *
875 + * Set the pointers to the path and query string according to the input.
876 + *
877 + * @param w is the structure with the client request
878 + * @param s is the first address of the string.
879 + * @param ptr is the address of the separator.
880 + */
881 +static void web_client_set_path_query(struct web_client *w, char *s, char *ptr) {
882 + w->url_path_length = (size_t)(ptr -s);
883 +
884 + w->url_search_path = ptr;
885 +}
886 +
887 +/**
888 + * Split path query
889 + *
890 + * Do the separation between path and query string
891 + *
892 + * @param w is the structure with the client request
893 + * @param s is the string to parse
894 + */
895 +void web_client_split_path_query(struct web_client *w, char *s) {
896 + //I am assuming here that the separator character(?) is not encoded
897 + char *ptr = strchr(s, '?');
898 + if(ptr) {
899 + w->separator = '?';
900 + web_client_set_path_query(w, s, ptr);
901 + return;
902 + }
903 +
904 + //Here I test the second possibility, the URL is completely encoded by the user.
905 + //I am not using the strcasestr, because it is fastest to check %3f and compare
906 + //the next character.
907 + //We executed some tests with "encodeURI(uri);" described in https://www.w3schools.com/jsref/jsref_encodeuri.asp
908 + //on July 1st, 2019, that show us that URLs won't have '?','=' and '&' encoded, but we decided to move in front
909 + //with the next part, because users can develop their own encoded that won't follow this rule.
910 + char *moveme = s;
911 + while (moveme) {
912 + ptr = strchr(moveme, '%');
913 + if(ptr) {
914 + char *test = (ptr+1);
915 + if (!strncmp(test, "3f", 2) || !strncmp(test, "3F", 2)) {
916 + w->separator = *ptr;
917 + web_client_set_path_query(w, s, ptr);
918 + return;
919 + }
920 + ptr++;
921 + }
922 +
923 + moveme = ptr;
924 + }
925 +
926 + w->separator = 0x00;
927 + w->url_path_length = strlen(s);
928 + w->url_search_path = NULL;
929 +}
930 +
931 +/**
932 + * Request validate
933 + *
934 + * @param w is the structure with the client request
935 + *
936 + * @return It returns HTTP_VALIDATION_OK on success and another code present
937 + * in the enum HTTP_VALIDATION otherwise.
938 + */
939 +static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
940 + char *s = (char *)buffer_tostring(w->response.data), *encoded_url = NULL;
941 +
942 + size_t last_pos = w->header_parse_last_size;
943 +
944 + w->header_parse_tries++;
945 + w->header_parse_last_size = buffer_strlen(w->response.data);
946 +
947 + int is_it_valid;
948 + if(w->header_parse_tries > 1) {
949 + if(last_pos > 4) last_pos -= 4; // allow searching for \r\n\r\n
950 + else last_pos = 0;
951 +
952 + if(w->header_parse_last_size < last_pos)
953 + last_pos = 0;
954 +
955 + is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size);
956 + if(!is_it_valid) {
957 + if(w->header_parse_tries > 10) {
958 + info("Disabling slow client after %zu attempts to read the request (%zu bytes received)", w->header_parse_tries, buffer_strlen(w->response.data));
959 + w->header_parse_tries = 0;
960 + w->header_parse_last_size = 0;
961 + web_client_disable_wait_receive(w);
962 + return HTTP_VALIDATION_NOT_SUPPORTED;
963 + }
964 +
965 + return HTTP_VALIDATION_INCOMPLETE;
966 + }
967 +
968 + is_it_valid = 1;
969 + } else {
970 + last_pos = w->header_parse_last_size;
971 + is_it_valid = url_is_request_complete(s, &s[last_pos], w->header_parse_last_size);
972 + }
973 +
974 + s = web_client_valid_method(w, s);
975 + if (!s) {
976 w->header_parse_tries = 0;
977 w->header_parse_last_size = 0;
978 web_client_disable_wait_receive(w);
979 +
980 return HTTP_VALIDATION_NOT_SUPPORTED;
981 + } else if (!is_it_valid) {
982 + //Invalid request, we have more data after the end of message
983 + char *check = strstr((char *)buffer_tostring(w->response.data), "\r\n\r\n");
984 + if(check) {
985 + check += 4;
986 + if (*check) {
987 + w->header_parse_tries = 0;
988 + w->header_parse_last_size = 0;
989 + web_client_disable_wait_receive(w);
990 + return HTTP_VALIDATION_NOT_SUPPORTED;
991 + }
992 + }
993 +
994 + web_client_enable_wait_receive(w);
995 + return HTTP_VALIDATION_INCOMPLETE;
996 }
997
909 - // find the SPACE + "HTTP/"
910 - while(*s) {
911 - // find the next space
912 - while (*s && *s != ' ') s++;
998 + //After the method we have the path and query string together
999 + encoded_url = s;
1000
914 - // is it SPACE + "HTTP/" ?
915 - if(*s && !strncmp(s, " HTTP/", 6)) break;
916 - else s++;
917 - }
1001 + //we search for the position where we have " HTTP/", because it finishes the user request
1002 + s = url_find_protocol(s);
1003
1004 // incomplete requests
1005 if(unlikely(!*s)) {
@@ -925,6 +1010,10 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
1010 // we have the end of encoded_url - remember it
1011 char *ue = s;
1012
1013 + //Variables used to map the variables in the query string case it is present
1014 + int total_variables;
1015 + char *ptr_variables[WEB_FIELDS_MAX];
1016 +
1017 // make sure we have complete request
1018 // complete requests contain: \r\n\r\n
1019 while(*s) {
@@ -942,13 +1031,38 @@ static inline HTTP_VALIDATION http_request_validate(struct web_client *w) {
1031 // a valid complete HTTP request found
1032
1033 *ue = '\0';
945 - if(!url_decode_r(w->decoded_url, encoded_url, NETDATA_WEB_REQUEST_URL_SIZE + 1))
946 - return HTTP_VALIDATION_MALFORMED_URL;
1034 + if(w->mode != WEB_CLIENT_MODE_NORMAL) {
1035 + if(!url_decode_r(w->decoded_url, encoded_url, NETDATA_WEB_REQUEST_URL_SIZE + 1))
1036 + return HTTP_VALIDATION_MALFORMED_URL;
1037 + } else {
1038 + web_client_split_path_query(w, encoded_url);
1039 +
1040 + if (w->separator) {
1041 + *w->url_search_path = 0x00;
1042 + }
1043 +
1044 + if(!url_decode_r(w->decoded_url, encoded_url, NETDATA_WEB_REQUEST_URL_SIZE + 1))
1045 + return HTTP_VALIDATION_MALFORMED_URL;
1046 +
1047 + if (w->separator) {
1048 + *w->url_search_path = w->separator;
1049 +
1050 + char *from = (encoded_url + w->url_path_length);
1051 + total_variables = url_map_query_string(ptr_variables, from);
1052 +
1053 + if (url_parse_query_string(w->decoded_query_string, NETDATA_WEB_REQUEST_URL_SIZE + 1, ptr_variables, total_variables)) {
1054 + return HTTP_VALIDATION_MALFORMED_URL;
1055 + }
1056 + }
1057 + }
1058 *ue = ' ';
948 -
1059 +
1060 // copy the URL - we are going to overwrite parts of it
1061 // TODO -- ideally we we should avoid copying buffers around
1062 strncpyz(w->last_url, w->decoded_url, NETDATA_WEB_REQUEST_URL_SIZE);
1063 + if (w->separator) {
1064 + *w->url_search_path = 0x00;
1065 + }
1066 #ifdef ENABLE_HTTPS
1067 if ( (!web_client_check_unix(w)) && (netdata_srv_ctx) ) {
1068 if ((w->ssl.conn) && ((w->ssl.flags & NETDATA_SSL_NO_HANDSHAKE) && (netdata_use_ssl_on_http & NETDATA_SSL_FORCE) && (w->mode != WEB_CLIENT_MODE_STREAM)) ) {
web/server/web_client.h
+17
@@ -24,6 +24,18 @@ typedef enum web_client_mode {
24 WEB_CLIENT_MODE_STREAM = 3
25 } WEB_CLIENT_MODE;
26
27 +typedef enum {
28 + HTTP_VALIDATION_OK,
29 + HTTP_VALIDATION_NOT_SUPPORTED,
30 + HTTP_VALIDATION_MALFORMED_URL,
31 +#ifdef ENABLE_HTTPS
32 + HTTP_VALIDATION_INCOMPLETE,
33 + HTTP_VALIDATION_REDIRECT
34 +#else
35 + HTTP_VALIDATION_INCOMPLETE
36 +#endif
37 +} HTTP_VALIDATION;
38 +
39 typedef enum web_client_flags {
40 WEB_CLIENT_FLAG_DEAD = 1 << 1, // if set, this client is dead
41
@@ -131,8 +143,12 @@ struct web_client {
143 char client_port[NI_MAXSERV+1];
144
145 char decoded_url[NETDATA_WEB_REQUEST_URL_SIZE + 1]; // we decode the URL in this buffer
146 + char decoded_query_string[NETDATA_WEB_REQUEST_URL_SIZE + 1]; // we decode the Query String in this buffer
147 char last_url[NETDATA_WEB_REQUEST_URL_SIZE+1]; // we keep a copy of the decoded URL here
148 char host[256];
149 + size_t url_path_length;
150 + char separator; // This value can be either '?' or 'f'
151 + char *url_search_path; //A pointer to the search path sent by the client
152
153 struct timeval tv_in, tv_ready;
154
@@ -162,6 +178,7 @@ struct web_client {
178 #endif
179 };
180
181 +
182 extern uid_t web_files_uid(void);
183 extern uid_t web_files_gid(void);
184