@cryptotaxi247 / netdata-1 / commits / b44d00bcb

journal: updates (#16150)

* dynamic facets hashtables; more facets fields for journal; enforce the facets blacklist * enable fstat caching per thread * enable fstat caching to speed up journal queries

Costa Tsaousis committed Oct 9, 2023 at 00:34 UTC b44d00bcbe21ac7002260777522e5ce7075dd001
3 files changed +248 -89
collectors/systemd-journal.plugin/README.md
+12 -5
@@ -98,6 +98,13 @@ For information about configuring a journals' centralization server, check [this
98
99 ## Journal Fields
100
101 +`systemd` journals are designed to support multiple fields per log entry. The power of `systemd` journals is that
102 +unlike other log management systems, it supports different, dynamic and variable fields for each log message
103 +while all fields are indexed for fast queries.
104 +
105 +This means that each application can log using its own unique fields.
106 +For a description of the most frequent fields found in `systemd` journals, check `man systemd.journal-fields`.
107 +
108 Fields found in the journal files are automatically added to the UI in multiple places to help you explore
109 and filter the data.
110
@@ -107,17 +114,17 @@ The plugin automatically enriches certain fields to make them more user-friendly
114 - `PRIORITY`: the numeric value is replaced with the human-readable name of each priority.
115 - `SYSLOG_FACILITY`: the encoded value is replaced with the human-readable name of each facility.
116 - `ERRNO`: the numeric value is annotated with the short name of each value.
110 -- `_UID` `_AUDIT_LOGINUID` and `_SYSTEMD_OWNER_UID`: the local user database is consulted to annotate them with usernames.
111 -- `_GID`: the local group database is consulted to annotate them with group names.
117 +- `_UID` `_AUDIT_LOGINUID`, `_SYSTEMD_OWNER_UID`, `OBJECT_UID`, `OBJECT_SYSTEMD_OWNER_UID`, `OBJECT_AUDIT_LOGINUID`: the local user database is consulted to annotate them with usernames.
118 +- `_GID`, `OBJECT_GID`: the local group database is consulted to annotate them with group names.
119 - `_CAP_EFFECTIVE`: the encoded value is annotated with a human-readable list of the linux capabilities.
120 - `_SOURCE_REALTIME_TIMESTAMP`: the numeric value is annotated with human-readable datetime in UTC.
121
122 The values of all other fields are presented as found in the journals.
123
124 > IMPORTANT:
118 -> `_UID` `_AUDIT_LOGINUID`, `_SYSTEMD_OWNER_UID` and `_GID` annotations are added during presentation and are taken
119 -> from the server running the plugin. For `remote` sources, the names presented may not reflect the actual user and
120 -> group names on the origin server. The numeric value will still be visible though, as-is on the origin server.
125 +> The UID and GID annotations are added during presentation and are taken from the server running the plugin.
126 +> For `remote` sources, the names presented may not reflect the actual user and group names on the origin server.
127 +> The numeric value will still be visible though, as-is on the origin server.
128
129 The annotations are not searchable with full text search. They are only added for the presentation of the fields.
130
collectors/systemd-journal.plugin/systemd-journal.c
+169 -72
@@ -13,14 +13,20 @@
13 #include <systemd/sd-journal.h>
14 #include <syslog.h>
15
16 +/*
17 + * TODO
18 + *
19 + * _UDEV_DEVLINK is frequently set more than once per field - support multi-value faces
20 + *
21 + */
22 +
23 +
24 // ----------------------------------------------------------------------------
25 // fstat64 overloading to speed up libsystemd
26 // https://github.com/systemd/systemd/pull/29261
27
28 #define ND_SD_JOURNAL_OPEN_FLAGS (0)
29
22 -#ifdef HAVE_SD_JOURNAL_OPEN_FILES_FD
23 -
30 #include <dlfcn.h>
31 #include <sys/stat.h>
32
@@ -32,46 +38,47 @@ struct fdstat64_cache_entry {
38 struct stat64 stat;
39 int ret;
40 size_t cached_count;
41 + size_t session;
42 };
36 -struct fdstat64_cache_entry fstat64_cache[FSTAT_CACHE_MAX] = {0 };
43
38 -static void fstat_cache_enable(int fd) {
39 - if(fd >= 0 && fd < FSTAT_CACHE_MAX) {
40 - fstat64_cache[fd].enabled = true;
41 - fstat64_cache[fd].updated = false;
42 - fstat64_cache[fd].cached_count = 0;
43 - }
44 +struct fdstat64_cache_entry fstat64_cache[FSTAT_CACHE_MAX] = {0 };
45 +static __thread size_t fstat_thread_calls = 0;
46 +static __thread size_t fstat_thread_cached_responses = 0;
47 +static __thread bool enable_thread_fstat = false;
48 +static __thread size_t fstat_caching_thread_session = 0;
49 +static size_t fstat_caching_global_session = 0;
50 +
51 +static void fstat_cache_enable_on_thread(void) {
52 + fstat_caching_thread_session = __atomic_add_fetch(&fstat_caching_global_session, 1, __ATOMIC_ACQUIRE);
53 + enable_thread_fstat = true;
54 }
55
46 -static size_t fstat_cache_disable(int fd) {
47 - size_t cached_count = 0;
48 -
49 - if(fd >= 0 && fd < FSTAT_CACHE_MAX) {
50 - fstat64_cache[fd].enabled = false;
51 - fstat64_cache[fd].updated = false;
52 - cached_count = fstat64_cache[fd].cached_count;
53 - fstat64_cache[fd].cached_count = 0;
54 - }
55 -
56 - return cached_count;
56 +static void fstat_cache_disable_on_thread(void) {
57 + fstat_caching_thread_session = __atomic_add_fetch(&fstat_caching_global_session, 1, __ATOMIC_RELEASE);
58 + enable_thread_fstat = false;
59 }
60
59 -static size_t fstat_calls = 0;
60 -static size_t fstat_cached_responses = 0;
61 -
61 int fstat64(int fd, struct stat64 *buf) {
62 static int (*real_fstat)(int, struct stat64 *) = NULL;
63 if (!real_fstat)
64 real_fstat = dlsym(RTLD_NEXT, "fstat64");
65
67 - fstat_calls++;
66 + fstat_thread_calls++;
67 +
68 + if(fd >= 0 && fd < FSTAT_CACHE_MAX) {
69 + if(enable_thread_fstat && fstat64_cache[fd].session != fstat_caching_thread_session) {
70 + fstat64_cache[fd].session = fstat_caching_thread_session;
71 + fstat64_cache[fd].enabled = true;
72 + fstat64_cache[fd].updated = false;
73 + }
74
69 - if(fd >= 0 && fd < FSTAT_CACHE_MAX && fstat64_cache[fd].enabled && fstat64_cache[fd].updated) {
70 - fstat_cached_responses++;
71 - errno = fstat64_cache[fd].err_no;
72 - *buf = fstat64_cache[fd].stat;
73 - fstat64_cache[fd].cached_count++;
74 - return fstat64_cache[fd].ret;
75 + if(fstat64_cache[fd].enabled && fstat64_cache[fd].updated && fstat64_cache[fd].session == fstat_caching_thread_session) {
76 + fstat_thread_cached_responses++;
77 + errno = fstat64_cache[fd].err_no;
78 + *buf = fstat64_cache[fd].stat;
79 + fstat64_cache[fd].cached_count++;
80 + return fstat64_cache[fd].ret;
81 + }
82 }
83
84 int ret = real_fstat(fd, buf);
@@ -81,13 +88,12 @@ int fstat64(int fd, struct stat64 *buf) {
88 fstat64_cache[fd].updated = true;
89 fstat64_cache[fd].err_no = errno;
90 fstat64_cache[fd].stat = *buf;
91 + fstat64_cache[fd].session = fstat_caching_thread_session;
92 }
93
94 return ret;
95 }
96
89 -#endif // HAVE_SD_JOURNAL_OPEN_FILES_FD
90 -
97 // ----------------------------------------------------------------------------
98
99 #define FACET_MAX_VALUE_LENGTH 8192
@@ -130,47 +136,114 @@ int fstat64(int fd, struct stat64 *buf) {
136
137 #define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS \
138 "*MESSAGE*" \
133 - "|CODE_LINE" \
134 - "|*DOCUMENTATION*" \
135 - "|TID" \
139 "|*_RAW" \
140 + "|*_USEC" \
141 "|*_NSEC" \
142 "|*TIMESTAMP*" \
143 "|*_ID" \
144 "|*_ID_*" \
141 - "|*_PID" \
142 - "|*_TID" \
145 "|__*" \
146 ""
147
148 #define SYSTEMD_KEYS_INCLUDED_IN_FACETS \
147 - "_COMM" \
148 - "|CONTAINER_NAME" \
149 - "|CONTAINER_TAG" \
150 - "|_TRANSPORT" \
151 - "|SYSLOG_IDENTIFIER" \
152 - "|SYSLOG_FACILITY" \
149 + \
150 + /* --- USER JOURNAL FIELDS --- */ \
151 + \
152 + /* "|MESSAGE" */ \
153 + /* "|MESSAGE_ID" */ \
154 "|PRIORITY" \
154 - "|_SYSTEMD_UNIT" \
155 + "|CODE_FILE" \
156 + /* "|CODE_LINE" */ \
157 + "|CODE_FUNC" \
158 + "|ERRNO" \
159 + /* "|INVOCATION_ID" */ \
160 + /* "|USER_INVOCATION_ID" */ \
161 + "|SYSLOG_FACILITY" \
162 + "|SYSLOG_IDENTIFIER" \
163 + /* "|SYSLOG_PID" */ \
164 + /* "|SYSLOG_TIMESTAMP" */ \
165 + /* "|SYSLOG_RAW" */ \
166 + /* "!DOCUMENTATION" */ \
167 + /* "|TID" */ \
168 + "|UNIT" \
169 + "|USER_UNIT" \
170 + "|UNIT_RESULT" /* undocumented */ \
171 + \
172 + \
173 + /* --- TRUSTED JOURNAL FIELDS --- */ \
174 + \
175 + /* "|_PID" */ \
176 + "|_UID" \
177 + "|_GID" \
178 + "|_COMM" \
179 + /* "|_EXE" */ \
180 + /* "|_CMDLINE" */ \
181 + "|_CAP_EFFECTIVE" \
182 + /* "|_AUDIT_SESSION" */ \
183 + "|_AUDIT_LOGINUID" \
184 + "|_SYSTEMD_CGROUP" \
185 "|_SYSTEMD_SLICE" \
186 + "|_SYSTEMD_UNIT" \
187 "|_SYSTEMD_USER_UNIT" \
188 "|_SYSTEMD_USER_SLICE" \
189 + "|_SYSTEMD_SESSION" \
190 "|_SYSTEMD_OWNER_UID" \
159 - "|_UID" \
160 - "|_GID" \
161 - "|UNIT" \
162 - "|USER_UNIT" \
163 - "|IMAGE_NAME" \
164 - "|ERRNO" \
191 + "|_SELINUX_CONTEXT" \
192 + /* "|_SOURCE_REALTIME_TIMESTAMP" */ \
193 + "|_BOOT_ID" \
194 + "|_MACHINE_ID" \
195 + /* "|_SYSTEMD_INVOCATION_ID" */ \
196 + "|_HOSTNAME" \
197 + "|_TRANSPORT" \
198 + "|_STREAM_ID" \
199 + /* "|LINE_BREAK" */ \
200 "|_NAMESPACE" \
201 + "|_RUNTIME_SCOPE" \
202 + \
203 + \
204 + /* --- KERNEL JOURNAL FIELDS --- */ \
205 + \
206 + /* "|_KERNEL_DEVICE" */ \
207 + "|_KERNEL_SUBSYSTEM" \
208 + /* "|_UDEV_SYSNAME" */ \
209 + "|_UDEV_DEVNODE" \
210 + /* "|_UDEV_DEVLINK" */ \
211 + \
212 + \
213 + /* --- LOGGING ON BEHALF --- */ \
214 + \
215 + "|OBJECT_UID" \
216 + "|OBJECT_GID" \
217 + "|OBJECT_COMM" \
218 + /* "|OBJECT_EXE" */ \
219 + /* "|OBJECT_CMDLINE" */ \
220 + /* "|OBJECT_AUDIT_SESSION" */ \
221 + "|OBJECT_AUDIT_LOGINUID" \
222 + "|OBJECT_SYSTEMD_CGROUP" \
223 + "|OBJECT_SYSTEMD_SESSION" \
224 + "|OBJECT_SYSTEMD_OWNER_UID" \
225 + "|OBJECT_SYSTEMD_UNIT" \
226 + "|OBJECT_SYSTEMD_USER_UNIT" \
227 + \
228 + \
229 + /* --- CORE DUMPS --- */ \
230 + \
231 "|COREDUMP_COMM" \
232 "|COREDUMP_UNIT" \
233 "|COREDUMP_USER_UNIT" \
234 "|COREDUMP_SIGNAL_NAME" \
235 "|COREDUMP_CGROUP" \
171 - "|_HOSTNAME" \
172 - "|UNIT_RESULT" \
173 - "|_RUNTIME_SCOPE" \
236 + \
237 + \
238 + /* --- DOCKER --- */ \
239 + \
240 + "|CONTAINER_ID" \
241 + /* "|CONTAINER_ID_FULL" */ \
242 + "|CONTAINER_NAME" \
243 + "|CONTAINER_TAG" \
244 + "|IMAGE_NAME" /* undocumented */ \
245 + /* "|CONTAINER_PARTIAL_MESSAGE" */ \
246 + \
247 ""
248
249 static netdata_mutex_t stdout_mutex = NETDATA_MUTEX_INITIALIZER;
@@ -609,25 +682,17 @@ static ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_one_file(
682 sd_journal *j = NULL;
683 errno = 0;
684
612 -//#ifdef HAVE_SD_JOURNAL_OPEN_FILES_FD
613 -// int fd = open(filename, O_RDONLY);
614 -// fstat_cache_enable(fd);
615 -//
616 -// if(sd_journal_open_files_fd(&j, &fd, 1, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) {
617 -// fqs->cached_count += fstat_cache_disable(fd);
618 -// close(fd);
619 -// return ND_SD_JOURNAL_FAILED_TO_OPEN;
620 -// }
621 -//#else // !HAVE_SD_JOURNAL_OPEN_FILES_FD
685 + fstat_cache_enable_on_thread();
686
687 const char *paths[2] = {
688 [0] = filename,
689 [1] = NULL,
690 };
627 - if(sd_journal_open_files(&j, paths, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j)
628 - return ND_SD_JOURNAL_FAILED_TO_OPEN;
691
630 -//#endif // !HAVE_SD_JOURNAL_OPEN_FILES_FD
692 + if(sd_journal_open_files(&j, paths, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) {
693 + fstat_cache_disable_on_thread();
694 + return ND_SD_JOURNAL_FAILED_TO_OPEN;
695 + }
696
697 ND_SD_JOURNAL_STATUS status;
698 bool matches_filters = true;
@@ -653,11 +718,7 @@ static ND_SD_JOURNAL_STATUS netdata_systemd_journal_query_one_file(
718 status = ND_SD_JOURNAL_NO_FILE_MATCHED;
719
720 sd_journal_close(j);
656 -
657 -//#ifdef HAVE_SD_JOURNAL_OPEN_FILES_FD
658 -// fqs->cached_count += fstat_cache_disable(fd);
659 -// close(fd);
660 -//#endif
721 + fstat_cache_disable_on_thread();
722
723 return status;
724 }
@@ -689,6 +750,8 @@ static void buffer_json_journal_versions(BUFFER *wb) {
750 }
751
752 static void journal_file_update_msg_ut(const char *filename, struct journal_file *jf) {
753 + fstat_cache_enable_on_thread();
754 +
755 const char *files[2] = {
756 [0] = filename,
757 [1] = NULL,
@@ -696,6 +759,8 @@ static void journal_file_update_msg_ut(const char *filename, struct journal_file
759
760 sd_journal *j = NULL;
761 if(sd_journal_open_files(&j, files, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) {
762 + fstat_cache_disable_on_thread();
763 +
764 if(!jf->logged_failure) {
765 netdata_log_error("cannot open journal file '%s', using file timestamps to understand time-frame.", filename);
766 jf->logged_failure = true;
@@ -719,6 +784,7 @@ static void journal_file_update_msg_ut(const char *filename, struct journal_file
784 }
785
786 sd_journal_close(j);
787 + fstat_cache_disable_on_thread();
788
789 if(first_ut > last_ut) {
790 internal_error(true, "timestamps are flipped in file '%s'", filename);
@@ -1174,6 +1240,8 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
1240 fqs->file_working++;
1241 fqs->cached_count = 0;
1242
1243 + size_t fs_calls = fstat_thread_calls;
1244 + size_t fs_cached = fstat_thread_cached_responses;
1245 size_t rows_useful = fqs->rows_useful;
1246 size_t rows_read = fqs->rows_read;
1247 size_t bytes_read = fqs->bytes_read;
@@ -1185,6 +1253,8 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
1253 rows_read = fqs->rows_read - rows_read;
1254 bytes_read = fqs->bytes_read - bytes_read;
1255 matches_setup_ut = fqs->matches_setup_ut - matches_setup_ut;
1256 + fs_calls = fstat_thread_calls - fs_calls;
1257 + fs_cached = fstat_thread_cached_responses - fs_cached;
1258
1259 started_ut = ended_ut;
1260 ended_ut = now_monotonic_usec();
@@ -1209,6 +1279,8 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
1279 buffer_json_member_add_uint64(wb, "bytes_read", bytes_read);
1280 buffer_json_member_add_double(wb, "bytes_per_second", (double) bytes_read / (double) duration_ut * (double) USEC_PER_SEC);
1281 buffer_json_member_add_uint64(wb, "duration_matches_ut", matches_setup_ut);
1282 + buffer_json_member_add_uint64(wb, "fstat_query_calls", fs_calls);
1283 + buffer_json_member_add_uint64(wb, "fstat_query_cached_responses", fs_cached);
1284 }
1285 buffer_json_object_close(wb); // journal file
1286
@@ -1290,6 +1362,13 @@ static int netdata_systemd_journal_query(BUFFER *wb, FACETS *facets, FUNCTION_QU
1362 facets_report(facets, wb, used_hashes_registry);
1363
1364 buffer_json_member_add_time_t(wb, "expires", now_realtime_sec() + (fqs->data_only ? 3600 : 0));
1365 +
1366 + buffer_json_member_add_object(wb, "_fstat_caching");
1367 + {
1368 + buffer_json_member_add_uint64(wb, "calls", fstat_thread_calls);
1369 + buffer_json_member_add_uint64(wb, "cached", fstat_thread_cached_responses);
1370 + }
1371 + buffer_json_object_close(wb); // _fstat_caching
1372 buffer_json_finalize(wb);
1373
1374 return HTTP_RESP_OK;
@@ -2035,6 +2114,8 @@ static void function_systemd_journal_progress(BUFFER *wb, const char *transactio
2114 }
2115
2116 static void function_systemd_journal(const char *transaction, char *function, int timeout, bool *cancelled) {
2117 + fstat_thread_calls = 0;
2118 + fstat_thread_cached_responses = 0;
2119 journal_files_registry_update();
2120
2121 BUFFER *wb = buffer_create(0, NULL);
@@ -2129,10 +2210,22 @@ static void function_systemd_journal(const char *transaction, char *function, in
2210 FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW,
2211 netdata_systemd_journal_transform_uid, NULL);
2212
2213 + facets_register_key_name_transformation(facets, "OBJECT_SYSTEMD_OWNER_UID",
2214 + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW,
2215 + netdata_systemd_journal_transform_uid, NULL);
2216 +
2217 + facets_register_key_name_transformation(facets, "OBJECT_UID",
2218 + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW,
2219 + netdata_systemd_journal_transform_uid, NULL);
2220 +
2221 facets_register_key_name_transformation(facets, "_GID",
2222 FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW,
2223 netdata_systemd_journal_transform_gid, NULL);
2224
2225 + facets_register_key_name_transformation(facets, "OBJECT_GID",
2226 + FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW,
2227 + netdata_systemd_journal_transform_gid, NULL);
2228 +
2229 facets_register_key_name_transformation(facets, "_CAP_EFFECTIVE",
2230 FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW,
2231 netdata_systemd_journal_transform_cap_effective, NULL);
@@ -2141,6 +2234,10 @@ static void function_systemd_journal(const char *transaction, char *function, in
2234 FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW,
2235 netdata_systemd_journal_transform_uid, NULL);
2236
2237 + facets_register_key_name_transformation(facets, "OBJECT_AUDIT_LOGINUID",
2238 + FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW,
2239 + netdata_systemd_journal_transform_uid, NULL);
2240 +
2241 facets_register_key_name_transformation(facets, "_SOURCE_REALTIME_TIMESTAMP",
2242 FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_TRANSFORM_VIEW,
2243 netdata_systemd_journal_transform_timestamp_usec, NULL);
@@ -2605,7 +2702,7 @@ int main(int argc __maybe_unused, char **argv __maybe_unused) {
2702
2703 if(argc == 2 && strcmp(argv[1], "debug") == 0) {
2704 bool cancelled = false;
2608 - char buf[] = "systemd-journal after:1696319393 before:1696320293 anchor:1696320283039944 direction:forward last:100 if_modified_since:1696320283039989 data_only:true delta:true tail:true slice:true source:all histogram:DHKucpqUoe1";
2705 + char buf[] = "systemd-journal after:-8000000 before:0 last:1";
2706 // char buf[] = "systemd-journal after:1695332964 before:1695937764 direction:backward last:100 slice:true source:all DHKucpqUoe1:PtVoyIuX.MU";
2707 // char buf[] = "systemd-journal after:1694511062 before:1694514662 anchor:1694514122024403";
2708 function_systemd_journal("123", buf, 600, &cancelled);
libnetdata/facets/facets.c
+67 -12
@@ -5,7 +5,7 @@
5 #define FACETS_KEYS_WITH_VALUES_MAX 200 // the max number of keys that can be facets
6 #define FACETS_KEYS_IN_ROW_MAX 500 // the max number of keys in a row
7
8 -#define FACETS_KEYS_HASHTABLE_ENTRIES 256
8 +#define FACETS_KEYS_HASHTABLE_ENTRIES 128
9 #define FACETS_VALUES_HASHTABLE_ENTRIES 32
10
11 // ----------------------------------------------------------------------------
@@ -139,8 +139,9 @@ struct facet_key {
139
140 struct {
141 bool enabled;
142 + uint32_t size;
143 uint32_t used;
143 - FACET_VALUE *hashtable[FACETS_VALUES_HASHTABLE_ENTRIES];
144 + FACET_VALUE **hashtable;
145 FACET_VALUE *ll;
146 } values;
147
@@ -194,7 +195,8 @@ struct facets {
195
196 struct {
197 size_t count;
197 - FACET_KEY *hashtable[FACETS_KEYS_HASHTABLE_ENTRIES];
198 + size_t size;
199 + FACET_KEY **hashtable;
200 FACET_KEY *ll;
201 } keys;
202
@@ -264,6 +266,7 @@ struct facets {
266 struct {
267 size_t registered;
268 size_t unique;
269 + size_t hashtable_increases;
270 } keys;
271
272 struct {
@@ -274,6 +277,7 @@ struct facets {
277 size_t indexed;
278 size_t inserts;
279 size_t conflicts;
280 + size_t hashtable_increases;
281 } values;
282
283 struct {
@@ -317,6 +321,8 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k);
321 static inline void FACETS_VALUES_INDEX_CREATE(FACET_KEY *k) {
322 k->values.ll = NULL;
323 k->values.used = 0;
324 + k->values.size = FACETS_VALUES_HASHTABLE_ENTRIES;
325 + k->values.hashtable = callocz(k->values.size, sizeof(FACET_VALUE *));
326 }
327
328 static inline void FACETS_VALUES_INDEX_DESTROY(FACET_KEY *k) {
@@ -329,8 +335,9 @@ static inline void FACETS_VALUES_INDEX_DESTROY(FACET_KEY *k) {
335 v = next;
336 }
337 k->values.ll = NULL;
338 + k->values.size = 0;
339 k->values.used = 0;
333 - memset(k->values.hashtable, 0, sizeof(k->values.hashtable));
340 + freez(k->values.hashtable);
341 k->values.enabled = false;
342 }
343
@@ -350,7 +357,7 @@ static inline void FACET_VALUE_ADD_CONFLICT(FACET_KEY *k, FACET_VALUE *v, const
357 }
358
359 static inline FACET_VALUE **facets_values_hashtable_slot(FACET_KEY *k, FACETS_HASH hash) {
353 - size_t slot = hash % FACETS_VALUES_HASHTABLE_ENTRIES;
360 + size_t slot = hash % k->values.size;
361 FACET_VALUE **v = &k->values.hashtable[slot];
362
363 while(*v && (*v)->hash != hash)
@@ -364,6 +371,19 @@ static inline FACET_VALUE *FACET_VALUE_GET_FROM_INDEX(FACET_KEY *k, FACETS_HASH
371 return *v_ptr;
372 }
373
374 +static void FACET_VALUES_HASHTABLE_DOUBLE(FACET_KEY *k) {
375 + // increase the hashtable size
376 + freez(k->values.hashtable);
377 + k->values.size *= 4;
378 + k->values.hashtable = callocz(k->values.size, sizeof(FACET_VALUE *));
379 + for(FACET_VALUE *v = k->values.ll ; v ;v = v->next) {
380 + FACET_VALUE **v_ptr = facets_values_hashtable_slot(k, v->hash);
381 + *v_ptr = v;
382 + v->parent_hashtable.next = NULL;
383 + }
384 + k->facets->operations.values.hashtable_increases++;
385 +}
386 +
387 static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VALUE * const tv) {
388 FACET_VALUE **v_ptr = facets_values_hashtable_slot(k, tv->hash);
389
@@ -382,6 +402,8 @@ static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VA
402
403 memcpy(v, tv, sizeof(*v));
404
405 + v->parent_hashtable.next = NULL; // make sure this is NULL
406 +
407 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(k->values.ll, v, prev, next);
408 k->values.used++;
409
@@ -396,6 +418,9 @@ static inline FACET_VALUE *FACET_VALUE_ADD_TO_INDEX(FACET_KEY *k, const FACET_VA
418
419 k->facets->operations.values.inserts++;
420
421 + if(unlikely(k->values.used > k->values.size / 2))
422 + FACET_VALUES_HASHTABLE_DOUBLE(k);
423 +
424 return v;
425 }
426
@@ -466,6 +491,8 @@ static inline void facet_key_late_init(FACETS *facets, FACET_KEY *k) {
491
492 static inline void FACETS_KEYS_INDEX_CREATE(FACETS *facets) {
493 facets->keys.ll = NULL;
494 + facets->keys.size = FACETS_KEYS_HASHTABLE_ENTRIES;
495 + facets->keys.hashtable = callocz(facets->keys.size, sizeof(FACET_KEY *));
496 facets->keys.count = 0;
497 facets->keys_with_values.used = 0;
498 }
@@ -482,7 +509,9 @@ static inline void FACETS_KEYS_INDEX_DESTROY(FACETS *facets) {
509
510 k = next;
511 }
485 - memset(facets->keys.hashtable, 0, sizeof(facets->keys.hashtable));
512 + freez(facets->keys.hashtable);
513 + facets->keys.hashtable = NULL;
514 + facets->keys.size = 0;
515 facets->keys.ll = NULL;
516 facets->keys.count = 0;
517 facets->keys_with_values.used = 0;
@@ -498,6 +527,19 @@ static inline FACET_KEY **facets_keys_hashtable_slot(FACETS *facets, FACETS_HASH
527 return k;
528 }
529
530 +static void FACET_KEYS_HASHTABLE_DOUBLE(FACETS *facets) {
531 + // increase the hashtable size
532 + freez(facets->keys.hashtable);
533 + facets->keys.size *= 4;
534 + facets->keys.hashtable = callocz(facets->keys.size, sizeof(FACET_KEY *));
535 + for(FACET_KEY *k = facets->keys.ll ; k ; k = k->next) {
536 + FACET_KEY **k_ptr = facets_keys_hashtable_slot(facets, k->hash);
537 + *k_ptr = k;
538 + k->parent_hashtable.next = NULL;
539 + }
540 + facets->operations.keys.hashtable_increases++;
541 +}
542 +
543 static inline FACET_KEY *FACETS_KEY_GET_FROM_INDEX(FACETS *facets, FACETS_HASH hash) {
544 FACET_KEY **k = facets_keys_hashtable_slot(facets, hash);
545 return *k;
@@ -577,6 +619,9 @@ static inline FACET_KEY *FACETS_KEY_CREATE(FACETS *facets, FACETS_HASH hash, con
619 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(facets->keys.ll, k, prev, next);
620 facets->keys.count++;
621
622 + if(facets->keys.count > facets->keys.size / 2)
623 + FACET_KEYS_HASHTABLE_DOUBLE(facets);
624 +
625 return k;
626 }
627
@@ -1244,16 +1289,18 @@ static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) {
1289 }
1290
1291 static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) {
1247 - bool included = true, excluded = false;
1292 + bool included = true, excluded = false, never = false;
1293
1294 if(k->options & (FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) {
1295 if(k->options & FACET_KEY_OPTION_FACET) {
1296 included = true;
1297 excluded = false;
1298 + never = false;
1299 }
1300 else if(k->options & (FACET_KEY_OPTION_NO_FACET | FACET_KEY_OPTION_NEVER_FACET)) {
1301 included = false;
1302 excluded = true;
1303 + never = true;
1304 }
1305 }
1306 else {
@@ -1263,8 +1310,10 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) {
1310 }
1311
1312 if (facets->excluded_keys) {
1266 - if (simple_pattern_matches(facets->excluded_keys, k->name))
1313 + if (simple_pattern_matches(facets->excluded_keys, k->name)) {
1314 excluded = true;
1315 + never = true;
1316 + }
1317 }
1318 }
1319
@@ -1276,6 +1325,10 @@ static inline bool facets_key_is_facet(FACETS *facets, FACET_KEY *k) {
1325
1326 k->options |= FACET_KEY_OPTION_NO_FACET;
1327 k->options &= ~FACET_KEY_OPTION_FACET;
1328 +
1329 + if(never)
1330 + k->options |= FACET_KEY_OPTION_NEVER_FACET;
1331 +
1332 return false;
1333 }
1334
@@ -1295,7 +1348,7 @@ FACETS *facets_create(uint32_t items_to_return, FACETS_OPTIONS options, const ch
1348 if(visible_keys && *visible_keys)
1349 facets->visible_keys = simple_pattern_create(visible_keys, "|", SIMPLE_PATTERN_EXACT, true);
1350
1298 - facets->max_items_to_return = items_to_return;
1351 + facets->max_items_to_return = items_to_return > 1 ? items_to_return : 2;
1352 facets->anchor.start_ut = 0;
1353 facets->anchor.stop_ut = 0;
1354 facets->anchor.direction = FACETS_ANCHOR_DIRECTION_BACKWARD;
@@ -1360,7 +1413,7 @@ void facets_set_query(FACETS *facets, const char *query) {
1413 }
1414
1415 void facets_set_items(FACETS *facets, uint32_t items) {
1363 - facets->max_items_to_return = items;
1416 + facets->max_items_to_return = items > 1 ? items : 2;
1417 }
1418
1419 void facets_set_anchor(FACETS *facets, usec_t start_ut, usec_t stop_ut, FACETS_ANCHOR_DIRECTION direction) {
@@ -2188,7 +2241,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
2241 RRDF_FIELD_TYPE_TIMESTAMP,
2242 RRDF_FIELD_VISUAL_VALUE,
2243 RRDF_FIELD_TRANSFORM_DATETIME_USEC, 0, NULL, NAN,
2191 - RRDF_FIELD_SORT_DESCENDING,
2244 + RRDF_FIELD_SORT_DESCENDING|RRDF_FIELD_SORT_FIXED,
2245 NULL,
2246 RRDF_FIELD_SUMMARY_COUNT,
2247 RRDF_FIELD_FILTER_RANGE,
@@ -2233,7 +2286,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
2286 RRDF_FIELD_TYPE_STRING,
2287 (k->options & FACET_KEY_OPTION_RICH_TEXT) ? RRDF_FIELD_VISUAL_RICH : RRDF_FIELD_VISUAL_VALUE,
2288 RRDF_FIELD_TRANSFORM_NONE, 0, NULL, NAN,
2236 - RRDF_FIELD_SORT_ASCENDING,
2289 + RRDF_FIELD_SORT_FIXED,
2290 NULL,
2291 RRDF_FIELD_SUMMARY_COUNT,
2292 (k->options & FACET_KEY_OPTION_NEVER_FACET) ? RRDF_FIELD_FILTER_NONE
@@ -2414,6 +2467,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
2467 {
2468 buffer_json_member_add_uint64(wb, "registered", facets->operations.keys.registered);
2469 buffer_json_member_add_uint64(wb, "unique", facets->operations.keys.unique);
2470 + buffer_json_member_add_uint64(wb, "hashtable_increases", facets->operations.keys.hashtable_increases);
2471 }
2472 buffer_json_object_close(wb); // keys
2473 buffer_json_member_add_object(wb, "values");
@@ -2425,6 +2479,7 @@ void facets_report(FACETS *facets, BUFFER *wb, DICTIONARY *used_hashes_registry)
2479 buffer_json_member_add_uint64(wb, "indexed", facets->operations.values.indexed);
2480 buffer_json_member_add_uint64(wb, "inserts", facets->operations.values.inserts);
2481 buffer_json_member_add_uint64(wb, "conflicts", facets->operations.values.conflicts);
2482 + buffer_json_member_add_uint64(wb, "hashtable_increases", facets->operations.values.hashtable_increases);
2483 }
2484 buffer_json_object_close(wb); // values
2485 buffer_json_member_add_object(wb, "fts");