convert invalid utf8 sequences to hex characters (#19333)
Costa Tsaousis committed
Jan 7, 2025 at 19:19 UTC
4437c51428e4b5ea61eb26d8e12aeb13f95ab26b
4 files changed
+55
-22
src/database/rrdlabels.c
+11
-3
@@ -1560,15 +1560,23 @@ int rrdlabels_unittest_sanitization() {
1560
1561
// invalid UTF8 No 1
1562
const unsigned char invalid1[] = { 0xC3, 0x28, 'A', 'B', 0x0 };
1563
- errors += rrdlabels_unittest_sanitize_value((const char *)invalid1, "(AB");
1563
+ errors += rrdlabels_unittest_sanitize_value((const char *)invalid1, "c3(AB");
1564
1565
// invalid UTF8 No 2
1566
const unsigned char invalid2[] = { 'A', 'B', 0xC3, 0x28, 'C', 'D', 0x0 };
1567
- errors += rrdlabels_unittest_sanitize_value((const char *)invalid2, "AB (CD");
1567
+ errors += rrdlabels_unittest_sanitize_value((const char *)invalid2, "ABc3(CD");
1568
1569
// invalid UTF8 No 3
1570
const unsigned char invalid3[] = { 'A', 'B', 0xC3, 0x28, 0x0 };
1571
- errors += rrdlabels_unittest_sanitize_value((const char *)invalid3, "AB (");
1571
+ errors += rrdlabels_unittest_sanitize_value((const char *)invalid3, "ABc3(");
1572
+
1573
+ // invalid UTF8 No 4
1574
+ const unsigned char invalid4[] = "clewd修改\xe7\x89";
1575
+ errors += rrdlabels_unittest_sanitize_value((const char *)invalid4, "clewd修改e789");
1576
+
1577
+ // invalid UTF8 No 5
1578
+ const unsigned char invalid5[] = "app.clewd修改\xe7\x89_fd_open_limits";
1579
+ errors += rrdlabels_unittest_sanitize_value((const char *)invalid5, "app.clewd修改e789_fd_open_limits");
1580
1581
return errors;
1582
}
src/libnetdata/buffer/buffer.c
+1
@@ -331,6 +331,7 @@ void buffer_json_finalize(BUFFER *wb) {
331
// ----------------------------------------------------------------------------
332
333
const char hex_digits[16] = "0123456789ABCDEF";
334
+const char hex_digits_lower[16] = "0123456789abcdef";
335
const char base64_digits[64] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
336
unsigned char hex_value_from_ascii[256];
337
unsigned char base64_value_from_ascii[256];
src/libnetdata/buffer/buffer.h
+1
@@ -11,6 +11,7 @@
11
#define BUFFER_JSON_MAX_DEPTH 32 // max is 255
12
13
extern const char hex_digits[16];
14
+extern const char hex_digits_lower[16];
15
extern const char base64_digits[64];
16
extern unsigned char hex_value_from_ascii[256];
17
extern unsigned char base64_value_from_ascii[256];
src/libnetdata/sanitizers/utf8-sanitizer.c
+42
-19
@@ -31,32 +31,55 @@ size_t text_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_si
31
while(*src && d < end) {
32
unsigned char c = *src;
33
34
- if(IS_UTF8_STARTBYTE(c) && IS_UTF8_BYTE(src[1]) && d + 2 <= end) {
35
- // UTF-8 multi-byte encoded character
34
+ if(IS_UTF8_STARTBYTE(c)) {
35
+ size_t utf8_character_bytes = 1;
36
+ bool valid_sequence = true;
37
37
- // find how big this character is (2-4 bytes)
38
- size_t utf_character_size = 2;
39
- while(utf_character_size < 4 &&
40
- d + utf_character_size <= end &&
41
- IS_UTF8_BYTE(src[utf_character_size]) &&
42
- !IS_UTF8_STARTBYTE(src[utf_character_size]))
43
- utf_character_size++;
38
+ // Determine expected sequence length based on start byte
39
+ if((c & 0xE0) == 0xC0) utf8_character_bytes = 2; // 2-byte sequence
40
+ else if((c & 0xF0) == 0xE0) utf8_character_bytes = 3; // 3-byte sequence
41
+ else if((c & 0xF8) == 0xF0) utf8_character_bytes = 4; // 4-byte sequence
42
+
43
+ if(utf8_character_bytes == 1)
44
+ valid_sequence = false;
45
+ else {
46
+ // make sure all the characters are valid
47
+ for(size_t i = 1; i < utf8_character_bytes; i++) {
48
+ if(!IS_UTF8_BYTE(src[i]) || IS_UTF8_STARTBYTE(src[i])) {
49
+ valid_sequence = false;
50
+ break;
51
+ }
52
+ }
53
+ }
54
55
if(utf) {
46
- while(utf_character_size) {
47
- utf_character_size--;
48
- *d++ = *src++;
56
+ if (valid_sequence && d + utf8_character_bytes <= end) {
57
+ // it is a valid utf8 character, and we have room at the destination
58
+ for (size_t i = 0; i < utf8_character_bytes; i++)
59
+ *d++ = *src++;
60
+ }
61
+ else {
62
+ *d++ = hex_digits_lower[(*src & 0xF0) >> 4];
63
+ if(d <= end) *d++ = hex_digits_lower[(*src & 0x0F)];
64
+
65
+ src++;
66
+
67
+ while(IS_UTF8_BYTE(*src) && !IS_UTF8_STARTBYTE(*src) && d <= end) {
68
+ *d++ = hex_digits_lower[(*src & 0xF0) >> 4];
69
+ if(d <= end) *d++ = hex_digits_lower[(*src & 0x0F)];
70
+ src++;
71
+ }
72
}
73
}
74
else {
52
- // UTF-8 characters are not allowed.
53
- // Assume it is an underscore
54
- // and skip all except the first byte
55
- *d++ = '_';
56
- src += (utf_character_size - 1);
57
- }
75
+ *d++ = '_'; // this fits, we tested in the while() above
76
59
- last_is_space = 0;
77
+ src++; // skip the utf8 start byte
78
+ // and skip the rest too
79
+ while(IS_UTF8_BYTE(*src) && !IS_UTF8_STARTBYTE(*src))
80
+ src++;
81
+ }
82
+ last_is_space = false;
83
mblen++;
84
continue;
85
}