Facets histograms (#15846)
Costa Tsaousis committed
Aug 21, 2023 at 11:20 UTC
41bd902426bd6e5241eb8a09f50fc22efa7f9c51
20 files changed
+7421
-87
Makefile.am
+1
@@ -201,6 +201,7 @@ LIBNETDATA_FILES = \
201
libnetdata/string/utf8.h \
202
libnetdata/worker_utilization/worker_utilization.c \
203
libnetdata/worker_utilization/worker_utilization.h \
204
+ libnetdata/xxhash.h \
205
libnetdata/http/http_defs.h \
206
libnetdata/dyn_conf/dyn_conf.c \
207
libnetdata/dyn_conf/dyn_conf.h \
REDISTRIBUTED.md
+5
@@ -190,4 +190,9 @@ connectivity is not available.
190
Copyright March 2010 by Université de Montréal, Richard Simard and Pierre L'Ecuyer
191
[GPL 3.0](https://www.gnu.org/licenses/gpl-3.0.en.html)
192
193
+- [xxHash](https://github.com/Cyan4973/xxHash)
194
+
195
+ Copyright (c) 2012-2021 Yann Collet
196
+ [BSD](https://github.com/Cyan4973/xxHash/blob/dev/LICENSE)
197
+
198
aclk/aclk_query.c
+1
-1
@@ -112,7 +112,7 @@ static int http_api_v2(struct aclk_query_thread *query_thr, aclk_query_t query)
112
if(web_client_timeout_checkpoint_and_check(w, &t)) {
113
netdata_log_access("QUERY CANCELED: QUEUE TIME EXCEEDED %llu ms (LIMIT %d ms)", t / USEC_PER_MS, query->timeout);
114
retval = 1;
115
- w->response.code = HTTP_RESP_BACKEND_FETCH_FAILED;
115
+ w->response.code = HTTP_RESP_SERVICE_UNAVAILABLE;
116
aclk_http_msg_v2_err(query_thr->client, query->callback_topic, query->msg_id, w->response.code, CLOUD_EC_SND_TIMEOUT, CLOUD_EMSG_SND_TIMEOUT, NULL, 0);
117
goto cleanup;
118
}
aclk/aclk_tx_msgs.c
+10
-9
@@ -194,15 +194,16 @@ int aclk_http_msg_v2(mqtt_wss_client client, const char *topic, const char *msg_
194
int rc = aclk_send_message_with_bin_payload(client, msg, topic, payload, payload_len);
195
196
switch (rc) {
197
- case HTTP_RESP_FORBIDDEN:
198
- aclk_http_msg_v2_err(client, topic, msg_id, rc, CLOUD_EC_REQ_REPLY_TOO_BIG, CLOUD_EMSG_REQ_REPLY_TOO_BIG, NULL, 0);
199
- break;
200
- case HTTP_RESP_INTERNAL_SERVER_ERROR:
201
- aclk_http_msg_v2_err(client, topic, msg_id, rc, CLOUD_EC_FAIL_TOPIC, CLOUD_EMSG_FAIL_TOPIC, payload, payload_len);
202
- break;
203
- case HTTP_RESP_BACKEND_FETCH_FAILED:
204
- aclk_http_msg_v2_err(client, topic, msg_id, rc, CLOUD_EC_SND_TIMEOUT, CLOUD_EMSG_SND_TIMEOUT, payload, payload_len);
205
- break;
197
+ case HTTP_RESP_FORBIDDEN:
198
+ aclk_http_msg_v2_err(client, topic, msg_id, rc, CLOUD_EC_REQ_REPLY_TOO_BIG, CLOUD_EMSG_REQ_REPLY_TOO_BIG, NULL, 0);
199
+ break;
200
+ case HTTP_RESP_INTERNAL_SERVER_ERROR:
201
+ aclk_http_msg_v2_err(client, topic, msg_id, rc, CLOUD_EC_FAIL_TOPIC, CLOUD_EMSG_FAIL_TOPIC, payload, payload_len);
202
+ break;
203
+ case HTTP_RESP_GATEWAY_TIMEOUT:
204
+ case HTTP_RESP_SERVICE_UNAVAILABLE:
205
+ aclk_http_msg_v2_err(client, topic, msg_id, rc, CLOUD_EC_SND_TIMEOUT, CLOUD_EMSG_SND_TIMEOUT, payload, payload_len);
206
+ break;
207
}
208
return rc ? rc : http_code;
209
}
collectors/plugins.d/pluginsd_parser.c
+5
-5
@@ -718,13 +718,13 @@ static void inflight_functions_insert_callback(const DICTIONARY_ITEM *item, void
718
string2str(pf->function));
719
720
// send the command to the plugin
721
- int ret = send_to_plugin(buffer, parser);
721
+ ssize_t ret = send_to_plugin(buffer, parser);
722
723
pf->sent_ut = now_realtime_usec();
724
725
if(ret < 0) {
726
- netdata_log_error("FUNCTION: failed to send function to plugin, error %d", ret);
727
- rrd_call_function_error(pf->destination_wb, "Failed to communicate with collector", HTTP_RESP_BACKEND_FETCH_FAILED);
726
+ netdata_log_error("FUNCTION '%s': failed to send it to the plugin, error %d", string2str(pf->function), ret);
727
+ rrd_call_function_error(pf->destination_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE);
728
}
729
else {
730
internal_error(LOG_FUNCTIONS,
@@ -740,8 +740,8 @@ static void inflight_functions_insert_callback(const DICTIONARY_ITEM *item, void
740
ret = send_to_plugin(pf->payload, parser);
741
742
if(ret < 0) {
743
- netdata_log_error("FUNCTION_PAYLOAD: failed to send function to plugin, error %d", ret);
744
- rrd_call_function_error(pf->destination_wb, "Failed to communicate with collector", HTTP_RESP_BACKEND_FETCH_FAILED);
743
+ netdata_log_error("FUNCTION_PAYLOAD '%s': failed to send function to plugin, error %d", string2str(pf->function), ret);
744
+ rrd_call_function_error(pf->destination_wb, "Failed to communicate with collector", HTTP_RESP_SERVICE_UNAVAILABLE);
745
}
746
else {
747
internal_error(LOG_FUNCTIONS,
collectors/systemd-journal.plugin/systemd-journal.c
+12
-1
@@ -29,6 +29,7 @@
29
#define JOURNAL_PARAMETER_ANCHOR "anchor"
30
#define JOURNAL_PARAMETER_LAST "last"
31
#define JOURNAL_PARAMETER_QUERY "query"
32
+#define JOURNAL_PARAMETER_HISTOGRAM "histogram"
33
34
#define SYSTEMD_ALWAYS_VISIBLE_KEYS NULL
35
#define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS NULL
@@ -80,8 +81,10 @@ int systemd_journal_query(BUFFER *wb, FACETS *facets, usec_t after_ut, usec_t be
81
r = sd_journal_open(&j, 0);
82
}
83
83
- if (r < 0)
84
+ if (r < 0) {
85
+ netdata_log_error("SYSTEMD-JOURNAL: Failed to open SystemD Journal, with error %d", r);
86
return HTTP_RESP_INTERNAL_SERVER_ERROR;
87
+ }
88
89
facets_rows_begin(facets);
90
@@ -350,6 +353,7 @@ static void function_systemd_journal(const char *transaction, char *function, ch
353
facets_accepted_param(facets, JOURNAL_PARAMETER_ANCHOR);
354
facets_accepted_param(facets, JOURNAL_PARAMETER_LAST);
355
facets_accepted_param(facets, JOURNAL_PARAMETER_QUERY);
356
+ facets_accepted_param(facets, JOURNAL_PARAMETER_HISTOGRAM);
357
358
// register the fields in the order you want them on the dashboard
359
@@ -379,6 +383,7 @@ static void function_systemd_journal(const char *transaction, char *function, ch
383
usec_t anchor = 0;
384
size_t last = 0;
385
const char *query = NULL;
386
+ const char *chart = NULL;
387
388
buffer_json_member_add_object(wb, "request");
389
buffer_json_member_add_object(wb, "filters");
@@ -406,6 +411,9 @@ static void function_systemd_journal(const char *transaction, char *function, ch
411
else if(strncmp(keyword, JOURNAL_PARAMETER_QUERY ":", strlen(JOURNAL_PARAMETER_QUERY ":")) == 0) {
412
query= &keyword[strlen(JOURNAL_PARAMETER_QUERY ":")];
413
}
414
+ else if(strncmp(keyword, JOURNAL_PARAMETER_HISTOGRAM ":", strlen(JOURNAL_PARAMETER_HISTOGRAM ":")) == 0) {
415
+ chart = &keyword[strlen(JOURNAL_PARAMETER_HISTOGRAM ":")];
416
+ }
417
else {
418
char *value = strchr(keyword, ':');
419
if(value) {
@@ -459,12 +467,15 @@ static void function_systemd_journal(const char *transaction, char *function, ch
467
buffer_json_member_add_uint64(wb, "anchor", anchor);
468
buffer_json_member_add_uint64(wb, "last", last);
469
buffer_json_member_add_string(wb, "query", query);
470
+ buffer_json_member_add_string(wb, "chart", chart);
471
buffer_json_member_add_time_t(wb, "timeout", timeout);
472
buffer_json_object_close(wb); // request
473
474
facets_set_items(facets, last);
475
facets_set_anchor(facets, anchor);
476
facets_set_query(facets, query);
477
+ facets_set_histogram(facets, chart ? chart : "PRIORITY", after_s * USEC_PER_SEC, before_s * USEC_PER_SEC);
478
+
479
int response = systemd_journal_query(wb, facets, after_s * USEC_PER_SEC, before_s * USEC_PER_SEC,
480
now_monotonic_usec() + (timeout - 1) * USEC_PER_SEC);
481
database/contexts/api_v2.c
+1
-1
@@ -2023,7 +2023,7 @@ int rrdcontext_to_json_v2(BUFFER *wb, struct api_v2_contexts_request *req, CONTE
2023
}
2024
else {
2025
buffer_strcat(wb, "query interrupted");
2026
- resp = HTTP_RESP_BACKEND_FETCH_FAILED;
2026
+ resp = HTTP_RESP_CLIENT_CLOSED_REQUEST;
2027
}
2028
goto cleanup;
2029
}
database/rrdfunctions.c
+1
-1
@@ -587,7 +587,7 @@ static int rrd_call_function_find(RRDHOST *host, BUFFER *wb, const char *name, s
587
return rrd_call_function_error(wb, "No collector is supplying this function on this host at this time.", HTTP_RESP_NOT_FOUND);
588
589
if(!(*rdcf)->collector->running)
590
- return rrd_call_function_error(wb, "The collector that registered this function, is not currently running.", HTTP_RESP_BACKEND_FETCH_FAILED);
590
+ return rrd_call_function_error(wb, "The collector that registered this function, is not currently running.", HTTP_RESP_SERVICE_UNAVAILABLE);
591
592
return HTTP_RESP_OK;
593
}
libnetdata/buffer/buffer.h
+2
-8
@@ -252,17 +252,11 @@ static inline void buffer_strcat(BUFFER *wb, const char *txt) {
252
static inline void buffer_strncat(BUFFER *wb, const char *txt, size_t len) {
253
if(unlikely(!txt || !*txt)) return;
254
255
- const char *t = txt;
255
buffer_need_bytes(wb, len + 1);
257
- char *s = &wb->buffer[wb->len];
258
- char *d = s;
259
- const char *e = &wb->buffer[wb->len + len];
256
261
- while(*t && d < e)
262
- *d++ = *t++;
263
-
264
- wb->len += d - s;
257
+ memcpy(&wb->buffer[wb->len], txt, len);
258
259
+ wb->len += len;
260
wb->buffer[wb->len] = '\0';
261
262
buffer_overflow_check(wb);
libnetdata/facets/facets.c
+585
-45
@@ -6,55 +6,23 @@ static void facets_row_free(FACETS *facets __maybe_unused, FACET_ROW *row);
6
7
// ----------------------------------------------------------------------------
8
9
-time_t calculate_bar_width(time_t before, time_t after) {
10
- // Array of valid durations in seconds
11
- static time_t valid_durations[] = {
12
- 1,
13
- 15,
14
- 30,
15
- 1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes
16
- 1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours
17
- 1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days
18
- 1 * (30*86400) // months
19
- };
20
- static int array_size = sizeof(valid_durations) / sizeof(valid_durations[0]);
21
-
22
- time_t duration = before - after;
23
- time_t bar_width = 1;
24
-
25
- for (int i = array_size - 1; i >= 0; --i) {
26
- if (duration / valid_durations[i] >= HISTOGRAM_COLUMNS) {
27
- bar_width = valid_durations[i];
28
- break;
29
- }
30
- }
31
-
32
- return bar_width;
33
-}
34
-
35
-// ----------------------------------------------------------------------------
36
-
37
-static inline void uint32_to_char(uint32_t num, char *out) {
38
- static char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789";
9
+static inline void uint64_to_char(uint64_t num, char *out) {
10
+ static const char id_encoding_characters[64 + 1] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ.abcdefghijklmnopqrstuvwxyz_0123456789";
11
12
int i;
41
- for(i = 5; i >= 0; --i) {
13
+ for(i = 10; i >= 0; --i) {
14
out[i] = id_encoding_characters[num & 63];
15
num >>= 6;
16
}
45
- out[6] = '\0';
17
}
18
48
-inline void facets_string_hash(const char *src, char *out) {
49
- uint32_t hash1 = fnv1a_hash32(src);
50
- uint32_t hash2 = djb2_hash32(src);
51
- uint32_t hash3 = larson_hash32(src);
19
+inline void facets_string_hash(const char *src, size_t len, char *out) {
20
+ XXH128_hash_t hash = XXH128(src, len, 0);
21
53
- uint32_to_char(hash1, out);
54
- uint32_to_char(hash2, &out[6]);
55
- uint32_to_char(hash3, &out[12]);
22
+ uint64_to_char(hash.high64, out);
23
+ uint64_to_char(hash.low64, &out[11]); // Starts right after the first 64-bit encoded string
24
57
- out[18] = '\0';
25
+ out[FACET_STRING_HASH_SIZE - 1] = '\0';
26
}
27
28
// ----------------------------------------------------------------------------
@@ -66,6 +34,9 @@ typedef struct facet_value {
34
35
uint32_t rows_matching_facet_value;
36
uint32_t final_facet_value_counter;
37
+
38
+ uint32_t *histogram;
39
+ uint32_t min, max, sum;
40
} FACET_VALUE;
41
42
struct facet_key {
@@ -124,6 +95,15 @@ struct facets {
95
uint32_t max_items_to_return;
96
uint32_t order;
97
98
+ struct {
99
+ char *chart;
100
+ bool enabled;
101
+ uint32_t slots;
102
+ usec_t slot_width;
103
+ usec_t after_ut;
104
+ usec_t before_ut;
105
+ } histogram;
106
+
107
struct {
108
FACET_ROW *last_added;
109
@@ -143,6 +123,515 @@ struct facets {
123
124
// ----------------------------------------------------------------------------
125
126
+static usec_t calculate_histogram_bar_width(usec_t after_ut, usec_t before_ut) {
127
+ // Array of valid durations in seconds
128
+ static time_t valid_durations[] = {
129
+ 1,
130
+ 15,
131
+ 30,
132
+ 1 * 60, 2 * 60, 3 * 60, 5 * 60, 10 * 60, 15 * 60, 30 * 60, // minutes
133
+ 1 * 3600, 2 * 3600, 6 * 3600, 8 * 3600, 12 * 3600, // hours
134
+ 1 * 86400, 2 * 86400, 3 * 86400, 5 * 86400, 7 * 86400, 14 * 86400, // days
135
+ 1 * (30*86400) // months
136
+ };
137
+ static int array_size = sizeof(valid_durations) / sizeof(valid_durations[0]);
138
+
139
+ usec_t duration = before_ut - after_ut;
140
+ usec_t bar_width = 1 * 60;
141
+
142
+ for (int i = array_size - 1; i >= 0; --i) {
143
+ if (duration / (valid_durations[i] * 60) >= HISTOGRAM_COLUMNS) {
144
+ bar_width = valid_durations[i] * 60;
145
+ break;
146
+ }
147
+ }
148
+
149
+ return bar_width;
150
+}
151
+
152
+static inline usec_t facets_histogram_slot_baseline_ut(FACETS *facets, usec_t ut) {
153
+ usec_t delta = ut % facets->histogram.slot_width;
154
+ return ut - delta;
155
+}
156
+
157
+void facets_set_histogram(FACETS *facets, const char *chart, usec_t after_ut, usec_t before_ut) {
158
+ facets->histogram.enabled = true;
159
+ facets->histogram.chart = chart ? strdupz(chart) : NULL;
160
+ facets->histogram.slot_width = calculate_histogram_bar_width(after_ut, before_ut);
161
+ facets->histogram.after_ut = facets_histogram_slot_baseline_ut(facets, after_ut);
162
+ facets->histogram.before_ut = facets_histogram_slot_baseline_ut(facets, before_ut) + facets->histogram.slot_width;
163
+ facets->histogram.slots = (facets->histogram.before_ut - facets->histogram.after_ut) / facets->histogram.slot_width + 1;
164
+}
165
+
166
+static inline void facets_histogram_update_value(FACETS *facets, FACET_KEY *k, FACET_VALUE *v, usec_t usec) {
167
+ if(!facets->histogram.enabled)
168
+ return;
169
+
170
+ if(unlikely(!v->histogram))
171
+ v->histogram = callocz(facets->histogram.slots, sizeof(*v->histogram));
172
+
173
+ usec_t base_ut = facets_histogram_slot_baseline_ut(facets, usec);
174
+
175
+ if(base_ut < facets->histogram.after_ut)
176
+ base_ut = facets->histogram.after_ut;
177
+
178
+ if(base_ut > facets->histogram.before_ut)
179
+ base_ut = facets->histogram.before_ut;
180
+
181
+ uint32_t slot = (base_ut - facets->histogram.after_ut) / facets->histogram.slot_width;
182
+
183
+ if(unlikely(slot >= facets->histogram.slots))
184
+ slot = facets->histogram.slots - 1;
185
+
186
+ v->histogram[slot]++;
187
+}
188
+
189
+static inline void facets_histogram_value_names(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
190
+ buffer_json_member_add_array(wb, key);
191
+ {
192
+ FACET_VALUE *v;
193
+ dfe_start_read(k->values, v) {
194
+ if(unlikely(!v->histogram))
195
+ continue;
196
+
197
+ buffer_json_add_array_item_string(wb, v->name);
198
+ }
199
+ dfe_done(v);
200
+ }
201
+ buffer_json_array_close(wb); // key
202
+}
203
+
204
+static inline void facets_histogram_value_units(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
205
+ buffer_json_member_add_array(wb, key);
206
+ {
207
+ FACET_VALUE *v;
208
+ dfe_start_read(k->values, v) {
209
+ if(unlikely(!v->histogram))
210
+ continue;
211
+
212
+ buffer_json_add_array_item_string(wb, "events");
213
+ }
214
+ dfe_done(v);
215
+ }
216
+ buffer_json_array_close(wb); // key
217
+}
218
+
219
+static inline void facets_histogram_value_min(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
220
+ buffer_json_member_add_array(wb, key);
221
+ {
222
+ FACET_VALUE *v;
223
+ dfe_start_read(k->values, v) {
224
+ if(unlikely(!v->histogram))
225
+ continue;
226
+
227
+ buffer_json_add_array_item_uint64(wb, v->min);
228
+ }
229
+ dfe_done(v);
230
+ }
231
+ buffer_json_array_close(wb); // key
232
+}
233
+
234
+static inline void facets_histogram_value_max(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
235
+ buffer_json_member_add_array(wb, key);
236
+ {
237
+ FACET_VALUE *v;
238
+ dfe_start_read(k->values, v) {
239
+ if(unlikely(!v->histogram))
240
+ continue;
241
+
242
+ buffer_json_add_array_item_uint64(wb, v->max);
243
+ }
244
+ dfe_done(v);
245
+ }
246
+ buffer_json_array_close(wb); // key
247
+}
248
+
249
+static inline void facets_histogram_value_avg(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
250
+ buffer_json_member_add_array(wb, key);
251
+ {
252
+ FACET_VALUE *v;
253
+ dfe_start_read(k->values, v) {
254
+ if(unlikely(!v->histogram))
255
+ continue;
256
+
257
+ buffer_json_add_array_item_double(wb, (double)v->sum / (double)facets->histogram.slots);
258
+ }
259
+ dfe_done(v);
260
+ }
261
+ buffer_json_array_close(wb); // key
262
+}
263
+
264
+static inline void facets_histogram_value_arp(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key) {
265
+ buffer_json_member_add_array(wb, key);
266
+ {
267
+ FACET_VALUE *v;
268
+ dfe_start_read(k->values, v) {
269
+ if(unlikely(!v->histogram))
270
+ continue;
271
+
272
+ buffer_json_add_array_item_uint64(wb, 0);
273
+ }
274
+ dfe_done(v);
275
+ }
276
+ buffer_json_array_close(wb); // key
277
+}
278
+
279
+static inline void facets_histogram_value_con(BUFFER *wb, FACETS *facets __maybe_unused, FACET_KEY *k, const char *key, uint32_t sum) {
280
+ buffer_json_member_add_array(wb, key);
281
+ {
282
+ FACET_VALUE *v;
283
+ dfe_start_read(k->values, v) {
284
+ if(unlikely(!v->histogram))
285
+ continue;
286
+
287
+ buffer_json_add_array_item_double(wb, (double)v->sum * 100.0 / (double)sum);
288
+ }
289
+ dfe_done(v);
290
+ }
291
+ buffer_json_array_close(wb); // key
292
+}
293
+
294
+static void facets_histogram_generate(FACETS *facets, FACET_KEY *k, BUFFER *wb) {
295
+ size_t dimensions = 0;
296
+ uint32_t min = UINT32_MAX, max = 0, sum = 0, count = 0;
297
+
298
+ {
299
+ FACET_VALUE *v;
300
+ dfe_start_read(k->values, v){
301
+ if (unlikely(!v->histogram))
302
+ continue;
303
+
304
+ dimensions++;
305
+
306
+ v->min = UINT32_MAX;
307
+ v->max = 0;
308
+ v->sum = 0;
309
+
310
+ for(uint32_t i = 0; i < facets->histogram.slots ;i++) {
311
+ uint32_t n = v->histogram[i];
312
+
313
+ if(n < min)
314
+ min = n;
315
+
316
+ if(n > max)
317
+ max = n;
318
+
319
+ sum += n;
320
+ count++;
321
+
322
+ if(n < v->min)
323
+ v->min = n;
324
+
325
+ if(n > v->max)
326
+ v->max = n;
327
+
328
+ v->sum += n;
329
+ }
330
+ }
331
+ dfe_done(v);
332
+ }
333
+
334
+ if(!dimensions)
335
+ return;
336
+
337
+ buffer_json_member_add_object(wb, "summary");
338
+ {
339
+ buffer_json_member_add_array(wb, "nodes");
340
+ {
341
+ buffer_json_add_array_item_object(wb); // node
342
+ {
343
+ buffer_json_member_add_string(wb, "mg", "default");
344
+ buffer_json_member_add_string(wb, "nm", "facets.histogram");
345
+ buffer_json_member_add_uint64(wb, "ni", 0);
346
+ buffer_json_member_add_object(wb, "st");
347
+ {
348
+ buffer_json_member_add_uint64(wb, "ai", 0);
349
+ buffer_json_member_add_uint64(wb, "code", 200);
350
+ buffer_json_member_add_string(wb, "msg", "");
351
+ }
352
+ buffer_json_object_close(wb); // st
353
+ buffer_json_member_add_object(wb, "is");
354
+ {
355
+ buffer_json_member_add_uint64(wb, "sl", 1);
356
+ buffer_json_member_add_uint64(wb, "qr", 1);
357
+ }
358
+ buffer_json_object_close(wb); // is
359
+ buffer_json_member_add_object(wb, "ds");
360
+ {
361
+ buffer_json_member_add_uint64(wb, "sl", dimensions);
362
+ buffer_json_member_add_uint64(wb, "qr", dimensions);
363
+ }
364
+ buffer_json_object_close(wb); // ds
365
+ buffer_json_member_add_object(wb, "sts");
366
+ {
367
+ buffer_json_member_add_uint64(wb, "min", min);
368
+ buffer_json_member_add_uint64(wb, "max", max);
369
+ buffer_json_member_add_double(wb, "avg", (double)sum / (double)count);
370
+ buffer_json_member_add_double(wb, "con", 100.0);
371
+ }
372
+ buffer_json_object_close(wb); // sts
373
+ }
374
+ buffer_json_object_close(wb); // node
375
+ }
376
+ buffer_json_array_close(wb); // nodes
377
+
378
+ buffer_json_member_add_array(wb, "contexts");
379
+ {
380
+ buffer_json_add_array_item_object(wb); // context
381
+ {
382
+ buffer_json_member_add_string(wb, "id", "facets.histogram");
383
+ buffer_json_member_add_object(wb, "is");
384
+ {
385
+ buffer_json_member_add_uint64(wb, "sl", 1);
386
+ buffer_json_member_add_uint64(wb, "qr", 1);
387
+ }
388
+ buffer_json_object_close(wb); // is
389
+ buffer_json_member_add_object(wb, "ds");
390
+ {
391
+ buffer_json_member_add_uint64(wb, "sl", dimensions);
392
+ buffer_json_member_add_uint64(wb, "qr", dimensions);
393
+ }
394
+ buffer_json_object_close(wb); // ds
395
+ buffer_json_member_add_object(wb, "sts");
396
+ {
397
+ buffer_json_member_add_uint64(wb, "min", min);
398
+ buffer_json_member_add_uint64(wb, "max", max);
399
+ buffer_json_member_add_double(wb, "avg", (double)sum / (double)count);
400
+ buffer_json_member_add_double(wb, "con", 100.0);
401
+ }
402
+ buffer_json_object_close(wb); // sts
403
+ }
404
+ buffer_json_object_close(wb); // context
405
+ }
406
+ buffer_json_array_close(wb); // contexts
407
+
408
+ buffer_json_member_add_array(wb, "instances");
409
+ {
410
+ buffer_json_add_array_item_object(wb); // instance
411
+ {
412
+ buffer_json_member_add_string(wb, "id", "facets.histogram");
413
+ buffer_json_member_add_uint64(wb, "ni", 0);
414
+ buffer_json_member_add_object(wb, "ds");
415
+ {
416
+ buffer_json_member_add_uint64(wb, "sl", dimensions);
417
+ buffer_json_member_add_uint64(wb, "qr", dimensions);
418
+ }
419
+ buffer_json_object_close(wb); // ds
420
+ buffer_json_member_add_object(wb, "sts");
421
+ {
422
+ buffer_json_member_add_uint64(wb, "min", min);
423
+ buffer_json_member_add_uint64(wb, "max", max);
424
+ buffer_json_member_add_double(wb, "avg", (double)sum / (double)count);
425
+ buffer_json_member_add_double(wb, "con", 100.0);
426
+ }
427
+ buffer_json_object_close(wb); // sts
428
+ }
429
+ buffer_json_object_close(wb); // instance
430
+ }
431
+ buffer_json_array_close(wb); // instances
432
+
433
+ buffer_json_member_add_array(wb, "dimensions");
434
+ {
435
+ size_t pri = 0;
436
+ FACET_VALUE *v;
437
+ dfe_start_read(k->values, v) {
438
+ if(unlikely(!v->histogram))
439
+ continue;
440
+
441
+ buffer_json_add_array_item_object(wb); // dimension
442
+ {
443
+ buffer_json_member_add_string(wb, "id", v->name);
444
+ buffer_json_member_add_object(wb, "ds");
445
+ {
446
+ buffer_json_member_add_uint64(wb, "sl", 1);
447
+ buffer_json_member_add_uint64(wb, "qr", 1);
448
+ }
449
+ buffer_json_object_close(wb); // ds
450
+ buffer_json_member_add_object(wb, "sts");
451
+ {
452
+ buffer_json_member_add_uint64(wb, "min", v->min);
453
+ buffer_json_member_add_uint64(wb, "max", v->max);
454
+ buffer_json_member_add_double(wb, "avg", (double)v->sum / (double)facets->histogram.slots);
455
+ buffer_json_member_add_double(wb, "con", (double)v->sum * 100.0 / (double)sum);
456
+ }
457
+ buffer_json_object_close(wb); // sts
458
+ buffer_json_member_add_uint64(wb, "pri", pri++);
459
+ }
460
+ buffer_json_object_close(wb); // dimension
461
+ }
462
+ dfe_done(v);
463
+ }
464
+ buffer_json_array_close(wb); // dimensions
465
+
466
+ buffer_json_member_add_array(wb, "labels");
467
+ buffer_json_array_close(wb); // labels
468
+
469
+ buffer_json_member_add_array(wb, "alerts");
470
+ buffer_json_array_close(wb); // alerts
471
+ }
472
+ buffer_json_object_close(wb); // summary
473
+
474
+ buffer_json_member_add_object(wb, "totals");
475
+ {
476
+ buffer_json_member_add_object(wb, "nodes");
477
+ {
478
+ buffer_json_member_add_uint64(wb, "sl", 1);
479
+ buffer_json_member_add_uint64(wb, "qr", 1);
480
+ }
481
+ buffer_json_object_close(wb); // nodes;
482
+ buffer_json_member_add_object(wb, "contexts");
483
+ {
484
+ buffer_json_member_add_uint64(wb, "sl", 1);
485
+ buffer_json_member_add_uint64(wb, "qr", 1);
486
+ }
487
+ buffer_json_object_close(wb); // contexts;
488
+ buffer_json_member_add_object(wb, "dimensions");
489
+ {
490
+ buffer_json_member_add_uint64(wb, "sl", dimensions);
491
+ buffer_json_member_add_uint64(wb, "qr", dimensions);
492
+ }
493
+ buffer_json_object_close(wb); // contexts;
494
+ }
495
+ buffer_json_object_close(wb); // totals
496
+
497
+ buffer_json_member_add_object(wb, "result");
498
+ {
499
+ facets_histogram_value_names(wb, facets, k, "labels");
500
+
501
+ buffer_json_member_add_object(wb, "point");
502
+ {
503
+ buffer_json_member_add_uint64(wb, "value", 0);
504
+ buffer_json_member_add_uint64(wb, "arp", 1);
505
+ buffer_json_member_add_uint64(wb, "pa", 2);
506
+ }
507
+ buffer_json_object_close(wb); // point
508
+
509
+ buffer_json_member_add_array(wb, "data");
510
+ {
511
+ usec_t t = facets->histogram.after_ut;
512
+ for(uint32_t i = 0; i < facets->histogram.slots ;i++) {
513
+ buffer_json_add_array_item_array(wb); // row
514
+ {
515
+ buffer_json_add_array_item_time_ms(wb, t / USEC_PER_SEC);
516
+
517
+ FACET_VALUE *v;
518
+ dfe_start_read(k->values, v) {
519
+ if(unlikely(!v->histogram))
520
+ continue;
521
+
522
+ buffer_json_add_array_item_array(wb); // point
523
+
524
+ buffer_json_add_array_item_uint64(wb, v->histogram[i]);
525
+ buffer_json_add_array_item_uint64(wb, 0);
526
+ buffer_json_add_array_item_uint64(wb, 1);
527
+
528
+ buffer_json_array_close(wb); // point
529
+ }
530
+ dfe_done(v);
531
+ }
532
+ buffer_json_array_close(wb); // row
533
+
534
+ t += facets->histogram.slot_width;
535
+ }
536
+ }
537
+ buffer_json_array_close(wb); //data
538
+ }
539
+ buffer_json_object_close(wb); // result
540
+
541
+ buffer_json_member_add_object(wb, "db");
542
+ {
543
+ buffer_json_member_add_uint64(wb, "tiers", 1);
544
+ buffer_json_member_add_uint64(wb, "update_every", 1);
545
+ buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC);
546
+ buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC);
547
+ buffer_json_member_add_string(wb, "units", "events");
548
+ buffer_json_member_add_object(wb, "dimensions");
549
+ {
550
+ facets_histogram_value_names(wb, facets, k, "ids");
551
+ facets_histogram_value_units(wb, facets, k, "units");
552
+
553
+ buffer_json_member_add_object(wb, "sts");
554
+ {
555
+ facets_histogram_value_min(wb, facets, k, "min");
556
+ facets_histogram_value_max(wb, facets, k, "max");
557
+ facets_histogram_value_avg(wb, facets, k, "avg");
558
+ facets_histogram_value_arp(wb, facets, k, "arp");
559
+ facets_histogram_value_con(wb, facets, k, "con", sum);
560
+ }
561
+ buffer_json_object_close(wb); // sts
562
+ }
563
+ buffer_json_object_close(wb); // dimensions
564
+
565
+ buffer_json_member_add_array(wb, "per_tier");
566
+ {
567
+ buffer_json_add_array_item_object(wb); // tier0
568
+ {
569
+ buffer_json_member_add_uint64(wb, "tier", 0);
570
+ buffer_json_member_add_uint64(wb, "queries", 1);
571
+ buffer_json_member_add_uint64(wb, "points", count);
572
+ buffer_json_member_add_time_t(wb, "update_every", 1);
573
+ buffer_json_member_add_time_t(wb, "first_entry", facets->histogram.after_ut / USEC_PER_SEC);
574
+ buffer_json_member_add_time_t(wb, "last_entry", facets->histogram.before_ut / USEC_PER_SEC);
575
+ }
576
+ buffer_json_object_close(wb); // tier0
577
+ }
578
+ buffer_json_array_close(wb); // per_tier
579
+ }
580
+ buffer_json_object_close(wb); // db
581
+
582
+ buffer_json_member_add_object(wb, "view");
583
+ {
584
+ buffer_json_member_add_string(wb, "title", "Events Distribution");
585
+ buffer_json_member_add_time_t(wb, "update_every", 1);
586
+ buffer_json_member_add_time_t(wb, "after", facets->histogram.after_ut / USEC_PER_SEC);
587
+ buffer_json_member_add_time_t(wb, "before", facets->histogram.before_ut / USEC_PER_SEC);
588
+ buffer_json_member_add_string(wb, "units", "events");
589
+ buffer_json_member_add_string(wb, "chart_type", "stacked");
590
+ buffer_json_member_add_object(wb, "dimensions");
591
+ {
592
+ buffer_json_member_add_array(wb, "grouped_by");
593
+ {
594
+ buffer_json_add_array_item_string(wb, "dimension");
595
+ }
596
+ buffer_json_array_close(wb); // grouped_by
597
+
598
+ facets_histogram_value_names(wb, facets, k, "ids");
599
+ facets_histogram_value_names(wb, facets, k, "names");
600
+ facets_histogram_value_units(wb, facets, k, "units");
601
+
602
+ buffer_json_member_add_object(wb, "sts");
603
+ {
604
+ facets_histogram_value_min(wb, facets, k, "min");
605
+ facets_histogram_value_max(wb, facets, k, "max");
606
+ facets_histogram_value_avg(wb, facets, k, "avg");
607
+ facets_histogram_value_arp(wb, facets, k, "arp");
608
+ facets_histogram_value_con(wb, facets, k, "con", sum);
609
+ }
610
+ buffer_json_object_close(wb); // sts
611
+ }
612
+ buffer_json_object_close(wb); // dimensions
613
+
614
+ buffer_json_member_add_uint64(wb, "min", min);
615
+ buffer_json_member_add_uint64(wb, "max", max);
616
+ }
617
+ buffer_json_object_close(wb); // view
618
+
619
+ buffer_json_member_add_array(wb, "agents");
620
+ {
621
+ buffer_json_add_array_item_object(wb); // agent
622
+ {
623
+ buffer_json_member_add_string(wb, "mg", "default");
624
+ buffer_json_member_add_string(wb, "nm", "facets.histogram");
625
+ buffer_json_member_add_time_t(wb, "now", now_realtime_sec());
626
+ buffer_json_member_add_uint64(wb, "ai", 0);
627
+ }
628
+ buffer_json_object_close(wb); // agent
629
+ }
630
+ buffer_json_array_close(wb); // agents
631
+}
632
+
633
+// ----------------------------------------------------------------------------
634
+
635
static inline void facet_value_is_used(FACET_KEY *k, FACET_VALUE *v) {
636
if(!k->key_found_in_row)
637
v->rows_matching_facet_value++;
@@ -218,7 +707,7 @@ static bool facet_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_un
707
if(v->name)
708
facet_value_is_used(k, v);
709
221
- internal_fatal(v->name && strcmp(v->name, nv->name) != 0, "hash conflict: '%s' and '%s' have the same hash '%s'", v->name, nv->name,
710
+ internal_fatal(v->name && nv->name && strcmp(v->name, nv->name) != 0, "value hash conflict: '%s' and '%s' have the same hash '%s'", v->name, nv->name,
711
dictionary_acquired_item_name(item));
712
713
return false;
@@ -226,6 +715,7 @@ static bool facet_value_conflict_callback(const DICTIONARY_ITEM *item __maybe_un
715
716
static void facet_value_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *value, void *data __maybe_unused) {
717
FACET_VALUE *v = value;
718
+ freez(v->histogram);
719
freez((char *)v->name);
720
}
721
@@ -278,6 +768,9 @@ static bool facet_key_conflict_callback(const DICTIONARY_ITEM *item __maybe_unus
768
facet_key_late_init(facets, k);
769
}
770
771
+ internal_fatal(k->name && nk->name && strcmp(k->name, nk->name) != 0, "key hash conflict: '%s' and '%s' have the same hash '%s'", k->name, nk->name,
772
+ dictionary_acquired_item_name(item));
773
+
774
if(k->options & FACET_KEY_OPTION_REORDER) {
775
k->order = facets->order++;
776
k->options &= ~FACET_KEY_OPTION_REORDER;
@@ -337,6 +830,7 @@ void facets_destroy(FACETS *facets) {
830
facets_row_free(facets, r);
831
}
832
833
+ freez(facets->histogram.chart);
834
freez(facets);
835
}
836
@@ -354,7 +848,7 @@ inline FACET_KEY *facets_register_key(FACETS *facets, const char *key, FACET_KEY
848
.default_selected_for_values = true,
849
};
850
char hash[FACET_STRING_HASH_SIZE];
357
- facets_string_hash(tk.name, hash);
851
+ facets_string_hash(tk.name, strlen(key), hash);
852
return dictionary_set(facets->keys, hash, &tk, sizeof(tk));
853
}
854
@@ -414,7 +908,7 @@ static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY *
908
k->transform.cb(facets, k->current_value.b, k->transform.data);
909
910
if(!k->current_value.updated) {
417
- buffer_strcat(k->current_value.b, FACET_VALUE_UNSET);
911
+ buffer_fast_strcat(k->current_value.b, FACET_VALUE_UNSET, sizeof(FACET_VALUE_UNSET) - 1);
912
k->current_value.updated = true;
913
}
914
@@ -431,7 +925,7 @@ static inline void facets_check_value(FACETS *facets __maybe_unused, FACET_KEY *
925
FACET_VALUE tk = {
926
.name = buffer_tostring(k->current_value.b),
927
};
434
- facets_string_hash(tk.name, k->current_value.hash);
928
+ facets_string_hash(tk.name, buffer_strlen(k->current_value.b), k->current_value.hash);
929
dictionary_set(k->values, k->current_value.hash, &tk, sizeof(tk));
930
}
931
else {
@@ -652,10 +1146,13 @@ void facets_row_finished(FACETS *facets, usec_t usec) {
1146
if(counted_by == total_keys) {
1147
if(k->values) {
1148
if(!k->current_value.hash[0])
655
- facets_string_hash(buffer_tostring(k->current_value.b), k->current_value.hash);
1149
+ facets_string_hash(buffer_tostring(k->current_value.b), buffer_strlen(k->current_value.b), k->current_value.hash);
1150
1151
FACET_VALUE *v = dictionary_get(k->values, k->current_value.hash);
1152
v->final_facet_value_counter++;
1153
+
1154
+ if(selected_by == total_keys)
1155
+ facets_histogram_update_value(facets, k, v, usec);
1156
}
1157
1158
found++;
@@ -826,6 +1323,49 @@ void facets_report(FACETS *facets, BUFFER *wb) {
1323
buffer_json_member_add_array(wb, "default_charts");
1324
buffer_json_array_close(wb);
1325
1326
+ if(facets->histogram.enabled) {
1327
+ const char *first_histogram = NULL;
1328
+ buffer_json_member_add_array(wb, "available_histograms");
1329
+ {
1330
+ FACET_KEY *k;
1331
+ dfe_start_read(facets->keys, k) {
1332
+ if (!k->values)
1333
+ continue;
1334
+
1335
+ if(unlikely(!first_histogram))
1336
+ first_histogram = k_dfe.name;
1337
+
1338
+ buffer_json_add_array_item_object(wb);
1339
+ buffer_json_member_add_string(wb, "id", k_dfe.name);
1340
+ buffer_json_member_add_string(wb, "name", k->name);
1341
+ buffer_json_object_close(wb);
1342
+ }
1343
+ dfe_done(k);
1344
+ }
1345
+ buffer_json_array_close(wb);
1346
+
1347
+ {
1348
+ const char *id = facets->histogram.chart;
1349
+ FACET_KEY *k = dictionary_get(facets->keys, id);
1350
+ if(!k || !k->values) {
1351
+ id = first_histogram;
1352
+ k = dictionary_get(facets->keys, id);
1353
+ }
1354
+
1355
+ if(k && k->values) {
1356
+ buffer_json_member_add_object(wb, "histogram");
1357
+ {
1358
+ buffer_json_member_add_string(wb, "id", id);
1359
+ buffer_json_member_add_string(wb, "name", k->name);
1360
+ buffer_json_member_add_object(wb, "chart");
1361
+ facets_histogram_generate(facets, k, wb);
1362
+ buffer_json_object_close(wb);
1363
+ }
1364
+ buffer_json_object_close(wb); // histogram
1365
+ }
1366
+ }
1367
+ }
1368
+
1369
buffer_json_member_add_object(wb, "items");
1370
{
1371
buffer_json_member_add_uint64(wb, "evaluated", facets->operations.evaluated);
libnetdata/facets/facets.h
+3
-2
@@ -29,8 +29,8 @@ typedef struct facet_row {
29
typedef struct facets FACETS;
30
typedef struct facet_key FACET_KEY;
31
32
-#define FACET_STRING_HASH_SIZE 19
33
-void facets_string_hash(const char *src, char *out);
32
+#define FACET_STRING_HASH_SIZE 23
33
+void facets_string_hash(const char *src, size_t len, char *out);
34
35
typedef void (*facets_key_transformer_t)(FACETS *facets __maybe_unused, BUFFER *wb, void *data);
36
typedef void (*facet_dynamic_row_t)(FACETS *facets, BUFFER *json_array, FACET_ROW_KEY_VALUE *rkv, FACET_ROW *row, void *data);
@@ -55,6 +55,7 @@ void facets_set_query(FACETS *facets, const char *query);
55
void facets_set_items(FACETS *facets, uint32_t items);
56
void facets_set_anchor(FACETS *facets, usec_t anchor);
57
void facets_register_facet_filter(FACETS *facets, const char *key_id, char *value_ids, FACET_KEY_OPTIONS options);
58
+void facets_set_histogram(FACETS *facets, const char *chart, usec_t after_ut, usec_t before_ut);
59
60
void facets_add_key_value(FACETS *facets, const char *key, const char *value);
61
void facets_add_key_value_length(FACETS *facets, const char *key, const char *value, size_t value_len);
libnetdata/http/http_defs.h
+2
-2
@@ -22,12 +22,12 @@
22
#define HTTP_RESP_CONFLICT 409
23
#define HTTP_RESP_PRECOND_FAIL 412
24
#define HTTP_RESP_CONTENT_TOO_LONG 413
25
+#define HTTP_RESP_CLIENT_CLOSED_REQUEST 499 // nginx's enxtension to the standard
26
27
// HTTP_CODES 5XX Server Errors
28
#define HTTP_RESP_INTERNAL_SERVER_ERROR 500
29
#define HTTP_RESP_INTERNAL_SERVER_ERROR_STR "Internal Server Error"
29
-#define HTTP_RESP_BACKEND_FETCH_FAILED 503 // 503 is right
30
-#define HTTP_RESP_SERVICE_UNAVAILABLE 503 // 503 is right
30
+#define HTTP_RESP_SERVICE_UNAVAILABLE 503
31
#define HTTP_RESP_GATEWAY_TIMEOUT 504
32
#define HTTP_RESP_BACKEND_RESPONSE_INVALID 591
33
libnetdata/libnetdata.h
+4
@@ -798,6 +798,10 @@ void for_each_open_fd(OPEN_FD_ACTION action, OPEN_FD_EXCLUDE excluded_fds);
798
void netdata_cleanup_and_exit(int ret) NORETURN;
799
void send_statistics(const char *action, const char *action_result, const char *action_data);
800
extern char *netdata_configured_host_prefix;
801
+
802
+#define XXH_INLINE_ALL
803
+#include "xxhash.h"
804
+
805
#include "libjudy/src/Judy.h"
806
#include "july/july.h"
807
#include "os.h"
libnetdata/xxhash.h
new
+6773
@@ -0,0 +1,6773 @@
1
+/*
2
+ * xxHash - Extremely Fast Hash algorithm
3
+ * Header File
4
+ * Copyright (C) 2012-2023 Yann Collet
5
+ *
6
+ * BSD 2-Clause License (https://www.opensource.org/licenses/bsd-license.php)
7
+ *
8
+ * Redistribution and use in source and binary forms, with or without
9
+ * modification, are permitted provided that the following conditions are
10
+ * met:
11
+ *
12
+ * * Redistributions of source code must retain the above copyright
13
+ * notice, this list of conditions and the following disclaimer.
14
+ * * Redistributions in binary form must reproduce the above
15
+ * copyright notice, this list of conditions and the following disclaimer
16
+ * in the documentation and/or other materials provided with the
17
+ * distribution.
18
+ *
19
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
+ *
31
+ * You can contact the author at:
32
+ * - xxHash homepage: https://www.xxhash.com
33
+ * - xxHash source repository: https://github.com/Cyan4973/xxHash
34
+ */
35
+
36
+/*!
37
+ * @mainpage xxHash
38
+ *
39
+ * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed
40
+ * limits.
41
+ *
42
+ * It is proposed in four flavors, in three families:
43
+ * 1. @ref XXH32_family
44
+ * - Classic 32-bit hash function. Simple, compact, and runs on almost all
45
+ * 32-bit and 64-bit systems.
46
+ * 2. @ref XXH64_family
47
+ * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most
48
+ * 64-bit systems (but _not_ 32-bit systems).
49
+ * 3. @ref XXH3_family
50
+ * - Modern 64-bit and 128-bit hash function family which features improved
51
+ * strength and performance across the board, especially on smaller data.
52
+ * It benefits greatly from SIMD and 64-bit without requiring it.
53
+ *
54
+ * Benchmarks
55
+ * ---
56
+ * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04.
57
+ * The open source benchmark program is compiled with clang v10.0 using -O3 flag.
58
+ *
59
+ * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity |
60
+ * | -------------------- | ------- | ----: | ---------------: | ------------------: |
61
+ * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 |
62
+ * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 |
63
+ * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 |
64
+ * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 |
65
+ * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 |
66
+ * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 |
67
+ * | RAM sequential read | | N/A | 28.0 GB/s | N/A |
68
+ * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 |
69
+ * | City64 | | 64 | 22.0 GB/s | 76.6 |
70
+ * | T1ha2 | | 64 | 22.0 GB/s | 99.0 |
71
+ * | City128 | | 128 | 21.7 GB/s | 57.7 |
72
+ * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 |
73
+ * | XXH64() | | 64 | 19.4 GB/s | 71.0 |
74
+ * | SpookyHash | | 64 | 19.3 GB/s | 53.2 |
75
+ * | Mum | | 64 | 18.0 GB/s | 67.0 |
76
+ * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 |
77
+ * | XXH32() | | 32 | 9.7 GB/s | 71.9 |
78
+ * | City32 | | 32 | 9.1 GB/s | 66.0 |
79
+ * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 |
80
+ * | Murmur3 | | 32 | 3.9 GB/s | 56.1 |
81
+ * | SipHash* | | 64 | 3.0 GB/s | 43.2 |
82
+ * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 |
83
+ * | HighwayHash | | 64 | 1.4 GB/s | 6.0 |
84
+ * | FNV64 | | 64 | 1.2 GB/s | 62.7 |
85
+ * | Blake2* | | 256 | 1.1 GB/s | 5.1 |
86
+ * | SHA1* | | 160 | 0.8 GB/s | 5.6 |
87
+ * | MD5* | | 128 | 0.6 GB/s | 7.8 |
88
+ * @note
89
+ * - Hashes which require a specific ISA extension are noted. SSE2 is also noted,
90
+ * even though it is mandatory on x64.
91
+ * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic
92
+ * by modern standards.
93
+ * - Small data velocity is a rough average of algorithm's efficiency for small
94
+ * data. For more accurate information, see the wiki.
95
+ * - More benchmarks and strength tests are found on the wiki:
96
+ * https://github.com/Cyan4973/xxHash/wiki
97
+ *
98
+ * Usage
99
+ * ------
100
+ * All xxHash variants use a similar API. Changing the algorithm is a trivial
101
+ * substitution.
102
+ *
103
+ * @pre
104
+ * For functions which take an input and length parameter, the following
105
+ * requirements are assumed:
106
+ * - The range from [`input`, `input + length`) is valid, readable memory.
107
+ * - The only exception is if the `length` is `0`, `input` may be `NULL`.
108
+ * - For C++, the objects must have the *TriviallyCopyable* property, as the
109
+ * functions access bytes directly as if it was an array of `unsigned char`.
110
+ *
111
+ * @anchor single_shot_example
112
+ * **Single Shot**
113
+ *
114
+ * These functions are stateless functions which hash a contiguous block of memory,
115
+ * immediately returning the result. They are the easiest and usually the fastest
116
+ * option.
117
+ *
118
+ * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits()
119
+ *
120
+ * @code{.c}
121
+ * #include <string.h>
122
+ * #include "xxhash.h"
123
+ *
124
+ * // Example for a function which hashes a null terminated string with XXH32().
125
+ * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed)
126
+ * {
127
+ * // NULL pointers are only valid if the length is zero
128
+ * size_t length = (string == NULL) ? 0 : strlen(string);
129
+ * return XXH32(string, length, seed);
130
+ * }
131
+ * @endcode
132
+ *
133
+ * @anchor streaming_example
134
+ * **Streaming**
135
+ *
136
+ * These groups of functions allow incremental hashing of unknown size, even
137
+ * more than what would fit in a size_t.
138
+ *
139
+ * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset()
140
+ *
141
+ * @code{.c}
142
+ * #include <stdio.h>
143
+ * #include <assert.h>
144
+ * #include "xxhash.h"
145
+ * // Example for a function which hashes a FILE incrementally with XXH3_64bits().
146
+ * XXH64_hash_t hashFile(FILE* f)
147
+ * {
148
+ * // Allocate a state struct. Do not just use malloc() or new.
149
+ * XXH3_state_t* state = XXH3_createState();
150
+ * assert(state != NULL && "Out of memory!");
151
+ * // Reset the state to start a new hashing session.
152
+ * XXH3_64bits_reset(state);
153
+ * char buffer[4096];
154
+ * size_t count;
155
+ * // Read the file in chunks
156
+ * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) {
157
+ * // Run update() as many times as necessary to process the data
158
+ * XXH3_64bits_update(state, buffer, count);
159
+ * }
160
+ * // Retrieve the finalized hash. This will not change the state.
161
+ * XXH64_hash_t result = XXH3_64bits_digest(state);
162
+ * // Free the state. Do not use free().
163
+ * XXH3_freeState(state);
164
+ * return result;
165
+ * }
166
+ * @endcode
167
+ *
168
+ * @file xxhash.h
169
+ * xxHash prototypes and implementation
170
+ */
171
+
172
+#if defined (__cplusplus)
173
+extern "C" {
174
+#endif
175
+
176
+/* ****************************
177
+ * INLINE mode
178
+ ******************************/
179
+/*!
180
+ * @defgroup public Public API
181
+ * Contains details on the public xxHash functions.
182
+ * @{
183
+ */
184
+#ifdef XXH_DOXYGEN
185
+/*!
186
+ * @brief Gives access to internal state declaration, required for static allocation.
187
+ *
188
+ * Incompatible with dynamic linking, due to risks of ABI changes.
189
+ *
190
+ * Usage:
191
+ * @code{.c}
192
+ * #define XXH_STATIC_LINKING_ONLY
193
+ * #include "xxhash.h"
194
+ * @endcode
195
+ */
196
+# define XXH_STATIC_LINKING_ONLY
197
+/* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */
198
+
199
+/*!
200
+ * @brief Gives access to internal definitions.
201
+ *
202
+ * Usage:
203
+ * @code{.c}
204
+ * #define XXH_STATIC_LINKING_ONLY
205
+ * #define XXH_IMPLEMENTATION
206
+ * #include "xxhash.h"
207
+ * @endcode
208
+ */
209
+# define XXH_IMPLEMENTATION
210
+/* Do not undef XXH_IMPLEMENTATION for Doxygen */
211
+
212
+/*!
213
+ * @brief Exposes the implementation and marks all functions as `inline`.
214
+ *
215
+ * Use these build macros to inline xxhash into the target unit.
216
+ * Inlining improves performance on small inputs, especially when the length is
217
+ * expressed as a compile-time constant:
218
+ *
219
+ * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
220
+ *
221
+ * It also keeps xxHash symbols private to the unit, so they are not exported.
222
+ *
223
+ * Usage:
224
+ * @code{.c}
225
+ * #define XXH_INLINE_ALL
226
+ * #include "xxhash.h"
227
+ * @endcode
228
+ * Do not compile and link xxhash.o as a separate object, as it is not useful.
229
+ */
230
+# define XXH_INLINE_ALL
231
+# undef XXH_INLINE_ALL
232
+/*!
233
+ * @brief Exposes the implementation without marking functions as inline.
234
+ */
235
+# define XXH_PRIVATE_API
236
+# undef XXH_PRIVATE_API
237
+/*!
238
+ * @brief Emulate a namespace by transparently prefixing all symbols.
239
+ *
240
+ * If you want to include _and expose_ xxHash functions from within your own
241
+ * library, but also want to avoid symbol collisions with other libraries which
242
+ * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix
243
+ * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE
244
+ * (therefore, avoid empty or numeric values).
245
+ *
246
+ * Note that no change is required within the calling program as long as it
247
+ * includes `xxhash.h`: Regular symbol names will be automatically translated
248
+ * by this header.
249
+ */
250
+# define XXH_NAMESPACE /* YOUR NAME HERE */
251
+# undef XXH_NAMESPACE
252
+#endif
253
+
254
+#if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
255
+ && !defined(XXH_INLINE_ALL_31684351384)
256
+ /* this section should be traversed only once */
257
+# define XXH_INLINE_ALL_31684351384
258
+ /* give access to the advanced API, required to compile implementations */
259
+# undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */
260
+# define XXH_STATIC_LINKING_ONLY
261
+ /* make all functions private */
262
+# undef XXH_PUBLIC_API
263
+# if defined(__GNUC__)
264
+# define XXH_PUBLIC_API static __inline __attribute__((unused))
265
+# elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
266
+# define XXH_PUBLIC_API static inline
267
+# elif defined(_MSC_VER)
268
+# define XXH_PUBLIC_API static __inline
269
+# else
270
+ /* note: this version may generate warnings for unused static functions */
271
+# define XXH_PUBLIC_API static
272
+# endif
273
+
274
+ /*
275
+ * This part deals with the special case where a unit wants to inline xxHash,
276
+ * but "xxhash.h" has previously been included without XXH_INLINE_ALL,
277
+ * such as part of some previously included *.h header file.
278
+ * Without further action, the new include would just be ignored,
279
+ * and functions would effectively _not_ be inlined (silent failure).
280
+ * The following macros solve this situation by prefixing all inlined names,
281
+ * avoiding naming collision with previous inclusions.
282
+ */
283
+ /* Before that, we unconditionally #undef all symbols,
284
+ * in case they were already defined with XXH_NAMESPACE.
285
+ * They will then be redefined for XXH_INLINE_ALL
286
+ */
287
+# undef XXH_versionNumber
288
+ /* XXH32 */
289
+# undef XXH32
290
+# undef XXH32_createState
291
+# undef XXH32_freeState
292
+# undef XXH32_reset
293
+# undef XXH32_update
294
+# undef XXH32_digest
295
+# undef XXH32_copyState
296
+# undef XXH32_canonicalFromHash
297
+# undef XXH32_hashFromCanonical
298
+ /* XXH64 */
299
+# undef XXH64
300
+# undef XXH64_createState
301
+# undef XXH64_freeState
302
+# undef XXH64_reset
303
+# undef XXH64_update
304
+# undef XXH64_digest
305
+# undef XXH64_copyState
306
+# undef XXH64_canonicalFromHash
307
+# undef XXH64_hashFromCanonical
308
+ /* XXH3_64bits */
309
+# undef XXH3_64bits
310
+# undef XXH3_64bits_withSecret
311
+# undef XXH3_64bits_withSeed
312
+# undef XXH3_64bits_withSecretandSeed
313
+# undef XXH3_createState
314
+# undef XXH3_freeState
315
+# undef XXH3_copyState
316
+# undef XXH3_64bits_reset
317
+# undef XXH3_64bits_reset_withSeed
318
+# undef XXH3_64bits_reset_withSecret
319
+# undef XXH3_64bits_update
320
+# undef XXH3_64bits_digest
321
+# undef XXH3_generateSecret
322
+ /* XXH3_128bits */
323
+# undef XXH128
324
+# undef XXH3_128bits
325
+# undef XXH3_128bits_withSeed
326
+# undef XXH3_128bits_withSecret
327
+# undef XXH3_128bits_reset
328
+# undef XXH3_128bits_reset_withSeed
329
+# undef XXH3_128bits_reset_withSecret
330
+# undef XXH3_128bits_reset_withSecretandSeed
331
+# undef XXH3_128bits_update
332
+# undef XXH3_128bits_digest
333
+# undef XXH128_isEqual
334
+# undef XXH128_cmp
335
+# undef XXH128_canonicalFromHash
336
+# undef XXH128_hashFromCanonical
337
+ /* Finally, free the namespace itself */
338
+# undef XXH_NAMESPACE
339
+
340
+ /* employ the namespace for XXH_INLINE_ALL */
341
+# define XXH_NAMESPACE XXH_INLINE_
342
+ /*
343
+ * Some identifiers (enums, type names) are not symbols,
344
+ * but they must nonetheless be renamed to avoid redeclaration.
345
+ * Alternative solution: do not redeclare them.
346
+ * However, this requires some #ifdefs, and has a more dispersed impact.
347
+ * Meanwhile, renaming can be achieved in a single place.
348
+ */
349
+# define XXH_IPREF(Id) XXH_NAMESPACE ## Id
350
+# define XXH_OK XXH_IPREF(XXH_OK)
351
+# define XXH_ERROR XXH_IPREF(XXH_ERROR)
352
+# define XXH_errorcode XXH_IPREF(XXH_errorcode)
353
+# define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t)
354
+# define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t)
355
+# define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
356
+# define XXH32_state_s XXH_IPREF(XXH32_state_s)
357
+# define XXH32_state_t XXH_IPREF(XXH32_state_t)
358
+# define XXH64_state_s XXH_IPREF(XXH64_state_s)
359
+# define XXH64_state_t XXH_IPREF(XXH64_state_t)
360
+# define XXH3_state_s XXH_IPREF(XXH3_state_s)
361
+# define XXH3_state_t XXH_IPREF(XXH3_state_t)
362
+# define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
363
+ /* Ensure the header is parsed again, even if it was previously included */
364
+# undef XXHASH_H_5627135585666179
365
+# undef XXHASH_H_STATIC_13879238742
366
+#endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
367
+
368
+/* ****************************************************************
369
+ * Stable API
370
+ *****************************************************************/
371
+#ifndef XXHASH_H_5627135585666179
372
+#define XXHASH_H_5627135585666179 1
373
+
374
+/*! @brief Marks a global symbol. */
375
+#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
376
+# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
377
+# ifdef XXH_EXPORT
378
+# define XXH_PUBLIC_API __declspec(dllexport)
379
+# elif XXH_IMPORT
380
+# define XXH_PUBLIC_API __declspec(dllimport)
381
+# endif
382
+# else
383
+# define XXH_PUBLIC_API /* do nothing */
384
+# endif
385
+#endif
386
+
387
+#ifdef XXH_NAMESPACE
388
+# define XXH_CAT(A,B) A##B
389
+# define XXH_NAME2(A,B) XXH_CAT(A,B)
390
+# define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
391
+/* XXH32 */
392
+# define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
393
+# define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
394
+# define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
395
+# define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
396
+# define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
397
+# define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
398
+# define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
399
+# define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
400
+# define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
401
+/* XXH64 */
402
+# define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
403
+# define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
404
+# define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
405
+# define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
406
+# define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
407
+# define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
408
+# define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
409
+# define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
410
+# define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
411
+/* XXH3_64bits */
412
+# define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
413
+# define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
414
+# define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
415
+# define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
416
+# define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
417
+# define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
418
+# define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
419
+# define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
420
+# define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
421
+# define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
422
+# define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
423
+# define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
424
+# define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
425
+# define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
426
+# define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
427
+/* XXH3_128bits */
428
+# define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
429
+# define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
430
+# define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
431
+# define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
432
+# define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
433
+# define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
434
+# define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
435
+# define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
436
+# define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
437
+# define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
438
+# define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
439
+# define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
440
+# define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
441
+# define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
442
+# define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
443
+#endif
444
+
445
+
446
+/* *************************************
447
+* Compiler specifics
448
+***************************************/
449
+
450
+/* specific declaration modes for Windows */
451
+#if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
452
+# if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
453
+# ifdef XXH_EXPORT
454
+# define XXH_PUBLIC_API __declspec(dllexport)
455
+# elif XXH_IMPORT
456
+# define XXH_PUBLIC_API __declspec(dllimport)
457
+# endif
458
+# else
459
+# define XXH_PUBLIC_API /* do nothing */
460
+# endif
461
+#endif
462
+
463
+#if defined (__GNUC__)
464
+# define XXH_CONSTF __attribute__((const))
465
+# define XXH_PUREF __attribute__((pure))
466
+# define XXH_MALLOCF __attribute__((malloc))
467
+#else
468
+# define XXH_CONSTF /* disable */
469
+# define XXH_PUREF
470
+# define XXH_MALLOCF
471
+#endif
472
+
473
+/* *************************************
474
+* Version
475
+***************************************/
476
+#define XXH_VERSION_MAJOR 0
477
+#define XXH_VERSION_MINOR 8
478
+#define XXH_VERSION_RELEASE 2
479
+/*! @brief Version number, encoded as two digits each */
480
+#define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
481
+
482
+/*!
483
+ * @brief Obtains the xxHash version.
484
+ *
485
+ * This is mostly useful when xxHash is compiled as a shared library,
486
+ * since the returned value comes from the library, as opposed to header file.
487
+ *
488
+ * @return @ref XXH_VERSION_NUMBER of the invoked library.
489
+ */
490
+XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void);
491
+
492
+
493
+/* ****************************
494
+* Common basic types
495
+******************************/
496
+#include <stddef.h> /* size_t */
497
+/*!
498
+ * @brief Exit code for the streaming API.
499
+ */
500
+typedef enum {
501
+ XXH_OK = 0, /*!< OK */
502
+ XXH_ERROR /*!< Error */
503
+} XXH_errorcode;
504
+
505
+
506
+/*-**********************************************************************
507
+* 32-bit hash
508
+************************************************************************/
509
+#if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */
510
+/*!
511
+ * @brief An unsigned 32-bit integer.
512
+ *
513
+ * Not necessarily defined to `uint32_t` but functionally equivalent.
514
+ */
515
+typedef uint32_t XXH32_hash_t;
516
+
517
+#elif !defined (__VMS) \
518
+ && (defined (__cplusplus) \
519
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
520
+# include <stdint.h>
521
+ typedef uint32_t XXH32_hash_t;
522
+
523
+#else
524
+# include <limits.h>
525
+# if UINT_MAX == 0xFFFFFFFFUL
526
+ typedef unsigned int XXH32_hash_t;
527
+# elif ULONG_MAX == 0xFFFFFFFFUL
528
+ typedef unsigned long XXH32_hash_t;
529
+# else
530
+# error "unsupported platform: need a 32-bit type"
531
+# endif
532
+#endif
533
+
534
+/*!
535
+ * @}
536
+ *
537
+ * @defgroup XXH32_family XXH32 family
538
+ * @ingroup public
539
+ * Contains functions used in the classic 32-bit xxHash algorithm.
540
+ *
541
+ * @note
542
+ * XXH32 is useful for older platforms, with no or poor 64-bit performance.
543
+ * Note that the @ref XXH3_family provides competitive speed for both 32-bit
544
+ * and 64-bit systems, and offers true 64/128 bit hash results.
545
+ *
546
+ * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families
547
+ * @see @ref XXH32_impl for implementation details
548
+ * @{
549
+ */
550
+
551
+/*!
552
+ * @brief Calculates the 32-bit hash of @p input using xxHash32.
553
+ *
554
+ * Speed on Core 2 Duo @ 3 GHz (single thread, SMHasher benchmark): 5.4 GB/s
555
+ *
556
+ * See @ref single_shot_example "Single Shot Example" for an example.
557
+ *
558
+ * @param input The block of data to be hashed, at least @p length bytes in size.
559
+ * @param length The length of @p input, in bytes.
560
+ * @param seed The 32-bit seed to alter the hash's output predictably.
561
+ *
562
+ * @pre
563
+ * The memory between @p input and @p input + @p length must be valid,
564
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
565
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
566
+ *
567
+ * @return The calculated 32-bit hash value.
568
+ *
569
+ * @see
570
+ * XXH64(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
571
+ * Direct equivalents for the other variants of xxHash.
572
+ * @see
573
+ * XXH32_createState(), XXH32_update(), XXH32_digest(): Streaming version.
574
+ */
575
+XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
576
+
577
+#ifndef XXH_NO_STREAM
578
+/*!
579
+ * Streaming functions generate the xxHash value from an incremental input.
580
+ * This method is slower than single-call functions, due to state management.
581
+ * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
582
+ *
583
+ * An XXH state must first be allocated using `XXH*_createState()`.
584
+ *
585
+ * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
586
+ *
587
+ * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
588
+ *
589
+ * The function returns an error code, with 0 meaning OK, and any other value
590
+ * meaning there is an error.
591
+ *
592
+ * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
593
+ * This function returns the nn-bits hash as an int or long long.
594
+ *
595
+ * It's still possible to continue inserting input into the hash state after a
596
+ * digest, and generate new hash values later on by invoking `XXH*_digest()`.
597
+ *
598
+ * When done, release the state using `XXH*_freeState()`.
599
+ *
600
+ * @see streaming_example at the top of @ref xxhash.h for an example.
601
+ */
602
+
603
+/*!
604
+ * @typedef struct XXH32_state_s XXH32_state_t
605
+ * @brief The opaque state struct for the XXH32 streaming API.
606
+ *
607
+ * @see XXH32_state_s for details.
608
+ */
609
+typedef struct XXH32_state_s XXH32_state_t;
610
+
611
+/*!
612
+ * @brief Allocates an @ref XXH32_state_t.
613
+ *
614
+ * Must be freed with XXH32_freeState().
615
+ * @return An allocated XXH32_state_t on success, `NULL` on failure.
616
+ */
617
+XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void);
618
+/*!
619
+ * @brief Frees an @ref XXH32_state_t.
620
+ *
621
+ * Must be allocated with XXH32_createState().
622
+ * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
623
+ * @return XXH_OK.
624
+ */
625
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
626
+/*!
627
+ * @brief Copies one @ref XXH32_state_t to another.
628
+ *
629
+ * @param dst_state The state to copy to.
630
+ * @param src_state The state to copy from.
631
+ * @pre
632
+ * @p dst_state and @p src_state must not be `NULL` and must not overlap.
633
+ */
634
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
635
+
636
+/*!
637
+ * @brief Resets an @ref XXH32_state_t to begin a new hash.
638
+ *
639
+ * This function resets and seeds a state. Call it before @ref XXH32_update().
640
+ *
641
+ * @param statePtr The state struct to reset.
642
+ * @param seed The 32-bit seed to alter the hash result predictably.
643
+ *
644
+ * @pre
645
+ * @p statePtr must not be `NULL`.
646
+ *
647
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
648
+ */
649
+XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed);
650
+
651
+/*!
652
+ * @brief Consumes a block of @p input to an @ref XXH32_state_t.
653
+ *
654
+ * Call this to incrementally consume blocks of data.
655
+ *
656
+ * @param statePtr The state struct to update.
657
+ * @param input The block of data to be hashed, at least @p length bytes in size.
658
+ * @param length The length of @p input, in bytes.
659
+ *
660
+ * @pre
661
+ * @p statePtr must not be `NULL`.
662
+ * @pre
663
+ * The memory between @p input and @p input + @p length must be valid,
664
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
665
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
666
+ *
667
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
668
+ */
669
+XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
670
+
671
+/*!
672
+ * @brief Returns the calculated hash value from an @ref XXH32_state_t.
673
+ *
674
+ * @note
675
+ * Calling XXH32_digest() will not affect @p statePtr, so you can update,
676
+ * digest, and update again.
677
+ *
678
+ * @param statePtr The state struct to calculate the hash from.
679
+ *
680
+ * @pre
681
+ * @p statePtr must not be `NULL`.
682
+ *
683
+ * @return The calculated xxHash32 value from that state.
684
+ */
685
+XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
686
+#endif /* !XXH_NO_STREAM */
687
+
688
+/******* Canonical representation *******/
689
+
690
+/*
691
+ * The default return values from XXH functions are unsigned 32 and 64 bit
692
+ * integers.
693
+ * This the simplest and fastest format for further post-processing.
694
+ *
695
+ * However, this leaves open the question of what is the order on the byte level,
696
+ * since little and big endian conventions will store the same number differently.
697
+ *
698
+ * The canonical representation settles this issue by mandating big-endian
699
+ * convention, the same convention as human-readable numbers (large digits first).
700
+ *
701
+ * When writing hash values to storage, sending them over a network, or printing
702
+ * them, it's highly recommended to use the canonical representation to ensure
703
+ * portability across a wider range of systems, present and future.
704
+ *
705
+ * The following functions allow transformation of hash values to and from
706
+ * canonical format.
707
+ */
708
+
709
+/*!
710
+ * @brief Canonical (big endian) representation of @ref XXH32_hash_t.
711
+ */
712
+typedef struct {
713
+ unsigned char digest[4]; /*!< Hash bytes, big endian */
714
+} XXH32_canonical_t;
715
+
716
+/*!
717
+ * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
718
+ *
719
+ * @param dst The @ref XXH32_canonical_t pointer to be stored to.
720
+ * @param hash The @ref XXH32_hash_t to be converted.
721
+ *
722
+ * @pre
723
+ * @p dst must not be `NULL`.
724
+ */
725
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
726
+
727
+/*!
728
+ * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t.
729
+ *
730
+ * @param src The @ref XXH32_canonical_t to convert.
731
+ *
732
+ * @pre
733
+ * @p src must not be `NULL`.
734
+ *
735
+ * @return The converted hash.
736
+ */
737
+XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
738
+
739
+
740
+/*! @cond Doxygen ignores this part */
741
+#ifdef __has_attribute
742
+# define XXH_HAS_ATTRIBUTE(x) __has_attribute(x)
743
+#else
744
+# define XXH_HAS_ATTRIBUTE(x) 0
745
+#endif
746
+/*! @endcond */
747
+
748
+/*! @cond Doxygen ignores this part */
749
+/*
750
+ * C23 __STDC_VERSION__ number hasn't been specified yet. For now
751
+ * leave as `201711L` (C17 + 1).
752
+ * TODO: Update to correct value when its been specified.
753
+ */
754
+#define XXH_C23_VN 201711L
755
+/*! @endcond */
756
+
757
+/*! @cond Doxygen ignores this part */
758
+/* C-language Attributes are added in C23. */
759
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute)
760
+# define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
761
+#else
762
+# define XXH_HAS_C_ATTRIBUTE(x) 0
763
+#endif
764
+/*! @endcond */
765
+
766
+/*! @cond Doxygen ignores this part */
767
+#if defined(__cplusplus) && defined(__has_cpp_attribute)
768
+# define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
769
+#else
770
+# define XXH_HAS_CPP_ATTRIBUTE(x) 0
771
+#endif
772
+/*! @endcond */
773
+
774
+/*! @cond Doxygen ignores this part */
775
+/*
776
+ * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute
777
+ * introduced in CPP17 and C23.
778
+ * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough
779
+ * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough
780
+ */
781
+#if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough)
782
+# define XXH_FALLTHROUGH [[fallthrough]]
783
+#elif XXH_HAS_ATTRIBUTE(__fallthrough__)
784
+# define XXH_FALLTHROUGH __attribute__ ((__fallthrough__))
785
+#else
786
+# define XXH_FALLTHROUGH /* fallthrough */
787
+#endif
788
+/*! @endcond */
789
+
790
+/*! @cond Doxygen ignores this part */
791
+/*
792
+ * Define XXH_NOESCAPE for annotated pointers in public API.
793
+ * https://clang.llvm.org/docs/AttributeReference.html#noescape
794
+ * As of writing this, only supported by clang.
795
+ */
796
+#if XXH_HAS_ATTRIBUTE(noescape)
797
+# define XXH_NOESCAPE __attribute__((noescape))
798
+#else
799
+# define XXH_NOESCAPE
800
+#endif
801
+/*! @endcond */
802
+
803
+
804
+/*!
805
+ * @}
806
+ * @ingroup public
807
+ * @{
808
+ */
809
+
810
+#ifndef XXH_NO_LONG_LONG
811
+/*-**********************************************************************
812
+* 64-bit hash
813
+************************************************************************/
814
+#if defined(XXH_DOXYGEN) /* don't include <stdint.h> */
815
+/*!
816
+ * @brief An unsigned 64-bit integer.
817
+ *
818
+ * Not necessarily defined to `uint64_t` but functionally equivalent.
819
+ */
820
+typedef uint64_t XXH64_hash_t;
821
+#elif !defined (__VMS) \
822
+ && (defined (__cplusplus) \
823
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
824
+# include <stdint.h>
825
+ typedef uint64_t XXH64_hash_t;
826
+#else
827
+# include <limits.h>
828
+# if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL
829
+ /* LP64 ABI says uint64_t is unsigned long */
830
+ typedef unsigned long XXH64_hash_t;
831
+# else
832
+ /* the following type must have a width of 64-bit */
833
+ typedef unsigned long long XXH64_hash_t;
834
+# endif
835
+#endif
836
+
837
+/*!
838
+ * @}
839
+ *
840
+ * @defgroup XXH64_family XXH64 family
841
+ * @ingroup public
842
+ * @{
843
+ * Contains functions used in the classic 64-bit xxHash algorithm.
844
+ *
845
+ * @note
846
+ * XXH3 provides competitive speed for both 32-bit and 64-bit systems,
847
+ * and offers true 64/128 bit hash results.
848
+ * It provides better speed for systems with vector processing capabilities.
849
+ */
850
+
851
+/*!
852
+ * @brief Calculates the 64-bit hash of @p input using xxHash64.
853
+ *
854
+ * This function usually runs faster on 64-bit systems, but slower on 32-bit
855
+ * systems (see benchmark).
856
+ *
857
+ * @param input The block of data to be hashed, at least @p length bytes in size.
858
+ * @param length The length of @p input, in bytes.
859
+ * @param seed The 64-bit seed to alter the hash's output predictably.
860
+ *
861
+ * @pre
862
+ * The memory between @p input and @p input + @p length must be valid,
863
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
864
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
865
+ *
866
+ * @return The calculated 64-bit hash.
867
+ *
868
+ * @see
869
+ * XXH32(), XXH3_64bits_withSeed(), XXH3_128bits_withSeed(), XXH128():
870
+ * Direct equivalents for the other variants of xxHash.
871
+ * @see
872
+ * XXH64_createState(), XXH64_update(), XXH64_digest(): Streaming version.
873
+ */
874
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
875
+
876
+/******* Streaming *******/
877
+#ifndef XXH_NO_STREAM
878
+/*!
879
+ * @brief The opaque state struct for the XXH64 streaming API.
880
+ *
881
+ * @see XXH64_state_s for details.
882
+ */
883
+typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
884
+
885
+/*!
886
+ * @brief Allocates an @ref XXH64_state_t.
887
+ *
888
+ * Must be freed with XXH64_freeState().
889
+ * @return An allocated XXH64_state_t on success, `NULL` on failure.
890
+ */
891
+XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void);
892
+
893
+/*!
894
+ * @brief Frees an @ref XXH64_state_t.
895
+ *
896
+ * Must be allocated with XXH64_createState().
897
+ * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState().
898
+ * @return XXH_OK.
899
+ */
900
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
901
+
902
+/*!
903
+ * @brief Copies one @ref XXH64_state_t to another.
904
+ *
905
+ * @param dst_state The state to copy to.
906
+ * @param src_state The state to copy from.
907
+ * @pre
908
+ * @p dst_state and @p src_state must not be `NULL` and must not overlap.
909
+ */
910
+XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state);
911
+
912
+/*!
913
+ * @brief Resets an @ref XXH64_state_t to begin a new hash.
914
+ *
915
+ * This function resets and seeds a state. Call it before @ref XXH64_update().
916
+ *
917
+ * @param statePtr The state struct to reset.
918
+ * @param seed The 64-bit seed to alter the hash result predictably.
919
+ *
920
+ * @pre
921
+ * @p statePtr must not be `NULL`.
922
+ *
923
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
924
+ */
925
+XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed);
926
+
927
+/*!
928
+ * @brief Consumes a block of @p input to an @ref XXH64_state_t.
929
+ *
930
+ * Call this to incrementally consume blocks of data.
931
+ *
932
+ * @param statePtr The state struct to update.
933
+ * @param input The block of data to be hashed, at least @p length bytes in size.
934
+ * @param length The length of @p input, in bytes.
935
+ *
936
+ * @pre
937
+ * @p statePtr must not be `NULL`.
938
+ * @pre
939
+ * The memory between @p input and @p input + @p length must be valid,
940
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
941
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
942
+ *
943
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
944
+ */
945
+XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
946
+
947
+/*!
948
+ * @brief Returns the calculated hash value from an @ref XXH64_state_t.
949
+ *
950
+ * @note
951
+ * Calling XXH64_digest() will not affect @p statePtr, so you can update,
952
+ * digest, and update again.
953
+ *
954
+ * @param statePtr The state struct to calculate the hash from.
955
+ *
956
+ * @pre
957
+ * @p statePtr must not be `NULL`.
958
+ *
959
+ * @return The calculated xxHash64 value from that state.
960
+ */
961
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr);
962
+#endif /* !XXH_NO_STREAM */
963
+/******* Canonical representation *******/
964
+
965
+/*!
966
+ * @brief Canonical (big endian) representation of @ref XXH64_hash_t.
967
+ */
968
+typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;
969
+
970
+/*!
971
+ * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t.
972
+ *
973
+ * @param dst The @ref XXH64_canonical_t pointer to be stored to.
974
+ * @param hash The @ref XXH64_hash_t to be converted.
975
+ *
976
+ * @pre
977
+ * @p dst must not be `NULL`.
978
+ */
979
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash);
980
+
981
+/*!
982
+ * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t.
983
+ *
984
+ * @param src The @ref XXH64_canonical_t to convert.
985
+ *
986
+ * @pre
987
+ * @p src must not be `NULL`.
988
+ *
989
+ * @return The converted hash.
990
+ */
991
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src);
992
+
993
+#ifndef XXH_NO_XXH3
994
+
995
+/*!
996
+ * @}
997
+ * ************************************************************************
998
+ * @defgroup XXH3_family XXH3 family
999
+ * @ingroup public
1000
+ * @{
1001
+ *
1002
+ * XXH3 is a more recent hash algorithm featuring:
1003
+ * - Improved speed for both small and large inputs
1004
+ * - True 64-bit and 128-bit outputs
1005
+ * - SIMD acceleration
1006
+ * - Improved 32-bit viability
1007
+ *
1008
+ * Speed analysis methodology is explained here:
1009
+ *
1010
+ * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
1011
+ *
1012
+ * Compared to XXH64, expect XXH3 to run approximately
1013
+ * ~2x faster on large inputs and >3x faster on small ones,
1014
+ * exact differences vary depending on platform.
1015
+ *
1016
+ * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic,
1017
+ * but does not require it.
1018
+ * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3
1019
+ * at competitive speeds, even without vector support. Further details are
1020
+ * explained in the implementation.
1021
+ *
1022
+ * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD
1023
+ * implementations for many common platforms:
1024
+ * - AVX512
1025
+ * - AVX2
1026
+ * - SSE2
1027
+ * - ARM NEON
1028
+ * - WebAssembly SIMD128
1029
+ * - POWER8 VSX
1030
+ * - s390x ZVector
1031
+ * This can be controlled via the @ref XXH_VECTOR macro, but it automatically
1032
+ * selects the best version according to predefined macros. For the x86 family, an
1033
+ * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c.
1034
+ *
1035
+ * XXH3 implementation is portable:
1036
+ * it has a generic C90 formulation that can be compiled on any platform,
1037
+ * all implementations generate exactly the same hash value on all platforms.
1038
+ * Starting from v0.8.0, it's also labelled "stable", meaning that
1039
+ * any future version will also generate the same hash value.
1040
+ *
1041
+ * XXH3 offers 2 variants, _64bits and _128bits.
1042
+ *
1043
+ * When only 64 bits are needed, prefer invoking the _64bits variant, as it
1044
+ * reduces the amount of mixing, resulting in faster speed on small inputs.
1045
+ * It's also generally simpler to manipulate a scalar return type than a struct.
1046
+ *
1047
+ * The API supports one-shot hashing, streaming mode, and custom secrets.
1048
+ */
1049
+/*-**********************************************************************
1050
+* XXH3 64-bit variant
1051
+************************************************************************/
1052
+
1053
+/*!
1054
+ * @brief 64-bit unseeded variant of XXH3.
1055
+ *
1056
+ * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of 0, however
1057
+ * it may have slightly better performance due to constant propagation of the
1058
+ * defaults.
1059
+ *
1060
+ * @see
1061
+ * XXH32(), XXH64(), XXH3_128bits(): equivalent for the other xxHash algorithms
1062
+ * @see
1063
+ * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants
1064
+ * @see
1065
+ * XXH3_64bits_reset(), XXH3_64bits_update(), XXH3_64bits_digest(): Streaming version.
1066
+ */
1067
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length);
1068
+
1069
+/*!
1070
+ * @brief 64-bit seeded variant of XXH3
1071
+ *
1072
+ * This variant generates a custom secret on the fly based on default secret
1073
+ * altered using the `seed` value.
1074
+ *
1075
+ * While this operation is decently fast, note that it's not completely free.
1076
+ *
1077
+ * @note
1078
+ * seed == 0 produces the same results as @ref XXH3_64bits().
1079
+ *
1080
+ * @param input The data to hash
1081
+ * @param length The length
1082
+ * @param seed The 64-bit seed to alter the state.
1083
+ */
1084
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
1085
+
1086
+/*!
1087
+ * The bare minimum size for a custom secret.
1088
+ *
1089
+ * @see
1090
+ * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(),
1091
+ * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret().
1092
+ */
1093
+#define XXH3_SECRET_SIZE_MIN 136
1094
+
1095
+/*!
1096
+ * @brief 64-bit variant of XXH3 with a custom "secret".
1097
+ *
1098
+ * It's possible to provide any blob of bytes as a "secret" to generate the hash.
1099
+ * This makes it more difficult for an external actor to prepare an intentional collision.
1100
+ * The main condition is that secretSize *must* be large enough (>= XXH3_SECRET_SIZE_MIN).
1101
+ * However, the quality of the secret impacts the dispersion of the hash algorithm.
1102
+ * Therefore, the secret _must_ look like a bunch of random bytes.
1103
+ * Avoid "trivial" or structured data such as repeated sequences or a text document.
1104
+ * Whenever in doubt about the "randomness" of the blob of bytes,
1105
+ * consider employing "XXH3_generateSecret()" instead (see below).
1106
+ * It will generate a proper high entropy secret derived from the blob of bytes.
1107
+ * Another advantage of using XXH3_generateSecret() is that
1108
+ * it guarantees that all bits within the initial blob of bytes
1109
+ * will impact every bit of the output.
1110
+ * This is not necessarily the case when using the blob of bytes directly
1111
+ * because, when hashing _small_ inputs, only a portion of the secret is employed.
1112
+ */
1113
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
1114
+
1115
+
1116
+/******* Streaming *******/
1117
+#ifndef XXH_NO_STREAM
1118
+/*
1119
+ * Streaming requires state maintenance.
1120
+ * This operation costs memory and CPU.
1121
+ * As a consequence, streaming is slower than one-shot hashing.
1122
+ * For better performance, prefer one-shot functions whenever applicable.
1123
+ */
1124
+
1125
+/*!
1126
+ * @brief The state struct for the XXH3 streaming API.
1127
+ *
1128
+ * @see XXH3_state_s for details.
1129
+ */
1130
+typedef struct XXH3_state_s XXH3_state_t;
1131
+XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void);
1132
+XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);
1133
+
1134
+/*!
1135
+ * @brief Copies one @ref XXH3_state_t to another.
1136
+ *
1137
+ * @param dst_state The state to copy to.
1138
+ * @param src_state The state to copy from.
1139
+ * @pre
1140
+ * @p dst_state and @p src_state must not be `NULL` and must not overlap.
1141
+ */
1142
+XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state);
1143
+
1144
+/*!
1145
+ * @brief Resets an @ref XXH3_state_t to begin a new hash.
1146
+ *
1147
+ * This function resets `statePtr` and generate a secret with default parameters. Call it before @ref XXH3_64bits_update().
1148
+ * Digest will be equivalent to `XXH3_64bits()`.
1149
+ *
1150
+ * @param statePtr The state struct to reset.
1151
+ *
1152
+ * @pre
1153
+ * @p statePtr must not be `NULL`.
1154
+ *
1155
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1156
+ *
1157
+ */
1158
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
1159
+
1160
+/*!
1161
+ * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
1162
+ *
1163
+ * This function resets `statePtr` and generate a secret from `seed`. Call it before @ref XXH3_64bits_update().
1164
+ * Digest will be equivalent to `XXH3_64bits_withSeed()`.
1165
+ *
1166
+ * @param statePtr The state struct to reset.
1167
+ * @param seed The 64-bit seed to alter the state.
1168
+ *
1169
+ * @pre
1170
+ * @p statePtr must not be `NULL`.
1171
+ *
1172
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1173
+ *
1174
+ */
1175
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
1176
+
1177
+/*!
1178
+ * XXH3_64bits_reset_withSecret():
1179
+ * `secret` is referenced, it _must outlive_ the hash streaming session.
1180
+ * Similar to one-shot API, `secretSize` must be >= `XXH3_SECRET_SIZE_MIN`,
1181
+ * and the quality of produced hash values depends on secret's entropy
1182
+ * (secret's content should look like a bunch of random bytes).
1183
+ * When in doubt about the randomness of a candidate `secret`,
1184
+ * consider employing `XXH3_generateSecret()` instead (see below).
1185
+ */
1186
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
1187
+
1188
+/*!
1189
+ * @brief Consumes a block of @p input to an @ref XXH3_state_t.
1190
+ *
1191
+ * Call this to incrementally consume blocks of data.
1192
+ *
1193
+ * @param statePtr The state struct to update.
1194
+ * @param input The block of data to be hashed, at least @p length bytes in size.
1195
+ * @param length The length of @p input, in bytes.
1196
+ *
1197
+ * @pre
1198
+ * @p statePtr must not be `NULL`.
1199
+ * @pre
1200
+ * The memory between @p input and @p input + @p length must be valid,
1201
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
1202
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
1203
+ *
1204
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1205
+ */
1206
+XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
1207
+
1208
+/*!
1209
+ * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t.
1210
+ *
1211
+ * @note
1212
+ * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update,
1213
+ * digest, and update again.
1214
+ *
1215
+ * @param statePtr The state struct to calculate the hash from.
1216
+ *
1217
+ * @pre
1218
+ * @p statePtr must not be `NULL`.
1219
+ *
1220
+ * @return The calculated XXH3 64-bit hash value from that state.
1221
+ */
1222
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
1223
+#endif /* !XXH_NO_STREAM */
1224
+
1225
+/* note : canonical representation of XXH3 is the same as XXH64
1226
+ * since they both produce XXH64_hash_t values */
1227
+
1228
+
1229
+/*-**********************************************************************
1230
+* XXH3 128-bit variant
1231
+************************************************************************/
1232
+
1233
+/*!
1234
+ * @brief The return value from 128-bit hashes.
1235
+ *
1236
+ * Stored in little endian order, although the fields themselves are in native
1237
+ * endianness.
1238
+ */
1239
+typedef struct {
1240
+ XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */
1241
+ XXH64_hash_t high64; /*!< `value >> 64` */
1242
+} XXH128_hash_t;
1243
+
1244
+/*!
1245
+ * @brief Unseeded 128-bit variant of XXH3
1246
+ *
1247
+ * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead
1248
+ * for shorter inputs.
1249
+ *
1250
+ * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of 0, however
1251
+ * it may have slightly better performance due to constant propagation of the
1252
+ * defaults.
1253
+ *
1254
+ * @see
1255
+ * XXH32(), XXH64(), XXH3_64bits(): equivalent for the other xxHash algorithms
1256
+ * @see
1257
+ * XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants
1258
+ * @see
1259
+ * XXH3_128bits_reset(), XXH3_128bits_update(), XXH3_128bits_digest(): Streaming version.
1260
+ */
1261
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len);
1262
+/*! @brief Seeded 128-bit variant of XXH3. @see XXH3_64bits_withSeed(). */
1263
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
1264
+/*! @brief Custom secret 128-bit variant of XXH3. @see XXH3_64bits_withSecret(). */
1265
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
1266
+
1267
+/******* Streaming *******/
1268
+#ifndef XXH_NO_STREAM
1269
+/*
1270
+ * Streaming requires state maintenance.
1271
+ * This operation costs memory and CPU.
1272
+ * As a consequence, streaming is slower than one-shot hashing.
1273
+ * For better performance, prefer one-shot functions whenever applicable.
1274
+ *
1275
+ * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
1276
+ * Use already declared XXH3_createState() and XXH3_freeState().
1277
+ *
1278
+ * All reset and streaming functions have same meaning as their 64-bit counterpart.
1279
+ */
1280
+
1281
+/*!
1282
+ * @brief Resets an @ref XXH3_state_t to begin a new hash.
1283
+ *
1284
+ * This function resets `statePtr` and generate a secret with default parameters. Call it before @ref XXH3_128bits_update().
1285
+ * Digest will be equivalent to `XXH3_128bits()`.
1286
+ *
1287
+ * @param statePtr The state struct to reset.
1288
+ *
1289
+ * @pre
1290
+ * @p statePtr must not be `NULL`.
1291
+ *
1292
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1293
+ *
1294
+ */
1295
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
1296
+
1297
+/*!
1298
+ * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
1299
+ *
1300
+ * This function resets `statePtr` and generate a secret from `seed`. Call it before @ref XXH3_128bits_update().
1301
+ * Digest will be equivalent to `XXH3_128bits_withSeed()`.
1302
+ *
1303
+ * @param statePtr The state struct to reset.
1304
+ * @param seed The 64-bit seed to alter the state.
1305
+ *
1306
+ * @pre
1307
+ * @p statePtr must not be `NULL`.
1308
+ *
1309
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1310
+ *
1311
+ */
1312
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
1313
+/*! @brief Custom secret 128-bit variant of XXH3. @see XXH_64bits_reset_withSecret(). */
1314
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
1315
+
1316
+/*!
1317
+ * @brief Consumes a block of @p input to an @ref XXH3_state_t.
1318
+ *
1319
+ * Call this to incrementally consume blocks of data.
1320
+ *
1321
+ * @param statePtr The state struct to update.
1322
+ * @param input The block of data to be hashed, at least @p length bytes in size.
1323
+ * @param length The length of @p input, in bytes.
1324
+ *
1325
+ * @pre
1326
+ * @p statePtr must not be `NULL`.
1327
+ * @pre
1328
+ * The memory between @p input and @p input + @p length must be valid,
1329
+ * readable, contiguous memory. However, if @p length is `0`, @p input may be
1330
+ * `NULL`. In C++, this also must be *TriviallyCopyable*.
1331
+ *
1332
+ * @return @ref XXH_OK on success, @ref XXH_ERROR on failure.
1333
+ */
1334
+XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
1335
+
1336
+/*!
1337
+ * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t.
1338
+ *
1339
+ * @note
1340
+ * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update,
1341
+ * digest, and update again.
1342
+ *
1343
+ * @param statePtr The state struct to calculate the hash from.
1344
+ *
1345
+ * @pre
1346
+ * @p statePtr must not be `NULL`.
1347
+ *
1348
+ * @return The calculated XXH3 128-bit hash value from that state.
1349
+ */
1350
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
1351
+#endif /* !XXH_NO_STREAM */
1352
+
1353
+/* Following helper functions make it possible to compare XXH128_hast_t values.
1354
+ * Since XXH128_hash_t is a structure, this capability is not offered by the language.
1355
+ * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
1356
+
1357
+/*!
1358
+ * XXH128_isEqual():
1359
+ * Return: 1 if `h1` and `h2` are equal, 0 if they are not.
1360
+ */
1361
+XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
1362
+
1363
+/*!
1364
+ * @brief Compares two @ref XXH128_hash_t
1365
+ * This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
1366
+ *
1367
+ * @return: >0 if *h128_1 > *h128_2
1368
+ * =0 if *h128_1 == *h128_2
1369
+ * <0 if *h128_1 < *h128_2
1370
+ */
1371
+XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2);
1372
+
1373
+
1374
+/******* Canonical representation *******/
1375
+typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;
1376
+
1377
+
1378
+/*!
1379
+ * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t.
1380
+ *
1381
+ * @param dst The @ref XXH128_canonical_t pointer to be stored to.
1382
+ * @param hash The @ref XXH128_hash_t to be converted.
1383
+ *
1384
+ * @pre
1385
+ * @p dst must not be `NULL`.
1386
+ */
1387
+XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash);
1388
+
1389
+/*!
1390
+ * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t.
1391
+ *
1392
+ * @param src The @ref XXH128_canonical_t to convert.
1393
+ *
1394
+ * @pre
1395
+ * @p src must not be `NULL`.
1396
+ *
1397
+ * @return The converted hash.
1398
+ */
1399
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src);
1400
+
1401
+
1402
+#endif /* !XXH_NO_XXH3 */
1403
+#endif /* XXH_NO_LONG_LONG */
1404
+
1405
+/*!
1406
+ * @}
1407
+ */
1408
+#endif /* XXHASH_H_5627135585666179 */
1409
+
1410
+
1411
+
1412
+#if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
1413
+#define XXHASH_H_STATIC_13879238742
1414
+/* ****************************************************************************
1415
+ * This section contains declarations which are not guaranteed to remain stable.
1416
+ * They may change in future versions, becoming incompatible with a different
1417
+ * version of the library.
1418
+ * These declarations should only be used with static linking.
1419
+ * Never use them in association with dynamic linking!
1420
+ ***************************************************************************** */
1421
+
1422
+/*
1423
+ * These definitions are only present to allow static allocation
1424
+ * of XXH states, on stack or in a struct, for example.
1425
+ * Never **ever** access their members directly.
1426
+ */
1427
+
1428
+/*!
1429
+ * @internal
1430
+ * @brief Structure for XXH32 streaming API.
1431
+ *
1432
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1433
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
1434
+ * an opaque type. This allows fields to safely be changed.
1435
+ *
1436
+ * Typedef'd to @ref XXH32_state_t.
1437
+ * Do not access the members of this struct directly.
1438
+ * @see XXH64_state_s, XXH3_state_s
1439
+ */
1440
+struct XXH32_state_s {
1441
+ XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */
1442
+ XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */
1443
+ XXH32_hash_t v[4]; /*!< Accumulator lanes */
1444
+ XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */
1445
+ XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */
1446
+ XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */
1447
+}; /* typedef'd to XXH32_state_t */
1448
+
1449
+
1450
+#ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */
1451
+
1452
+/*!
1453
+ * @internal
1454
+ * @brief Structure for XXH64 streaming API.
1455
+ *
1456
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1457
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
1458
+ * an opaque type. This allows fields to safely be changed.
1459
+ *
1460
+ * Typedef'd to @ref XXH64_state_t.
1461
+ * Do not access the members of this struct directly.
1462
+ * @see XXH32_state_s, XXH3_state_s
1463
+ */
1464
+struct XXH64_state_s {
1465
+ XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */
1466
+ XXH64_hash_t v[4]; /*!< Accumulator lanes */
1467
+ XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */
1468
+ XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */
1469
+ XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/
1470
+ XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */
1471
+}; /* typedef'd to XXH64_state_t */
1472
+
1473
+#ifndef XXH_NO_XXH3
1474
+
1475
+#if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
1476
+# include <stdalign.h>
1477
+# define XXH_ALIGN(n) alignas(n)
1478
+#elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
1479
+/* In C++ alignas() is a keyword */
1480
+# define XXH_ALIGN(n) alignas(n)
1481
+#elif defined(__GNUC__)
1482
+# define XXH_ALIGN(n) __attribute__ ((aligned(n)))
1483
+#elif defined(_MSC_VER)
1484
+# define XXH_ALIGN(n) __declspec(align(n))
1485
+#else
1486
+# define XXH_ALIGN(n) /* disabled */
1487
+#endif
1488
+
1489
+/* Old GCC versions only accept the attribute after the type in structures. */
1490
+#if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \
1491
+ && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \
1492
+ && defined(__GNUC__)
1493
+# define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
1494
+#else
1495
+# define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
1496
+#endif
1497
+
1498
+/*!
1499
+ * @brief The size of the internal XXH3 buffer.
1500
+ *
1501
+ * This is the optimal update size for incremental hashing.
1502
+ *
1503
+ * @see XXH3_64b_update(), XXH3_128b_update().
1504
+ */
1505
+#define XXH3_INTERNALBUFFER_SIZE 256
1506
+
1507
+/*!
1508
+ * @internal
1509
+ * @brief Default size of the secret buffer (and @ref XXH3_kSecret).
1510
+ *
1511
+ * This is the size used in @ref XXH3_kSecret and the seeded functions.
1512
+ *
1513
+ * Not to be confused with @ref XXH3_SECRET_SIZE_MIN.
1514
+ */
1515
+#define XXH3_SECRET_DEFAULT_SIZE 192
1516
+
1517
+/*!
1518
+ * @internal
1519
+ * @brief Structure for XXH3 streaming API.
1520
+ *
1521
+ * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1522
+ * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined.
1523
+ * Otherwise it is an opaque type.
1524
+ * Never use this definition in combination with dynamic library.
1525
+ * This allows fields to safely be changed in the future.
1526
+ *
1527
+ * @note ** This structure has a strict alignment requirement of 64 bytes!! **
1528
+ * Do not allocate this with `malloc()` or `new`,
1529
+ * it will not be sufficiently aligned.
1530
+ * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation.
1531
+ *
1532
+ * Typedef'd to @ref XXH3_state_t.
1533
+ * Do never access the members of this struct directly.
1534
+ *
1535
+ * @see XXH3_INITSTATE() for stack initialization.
1536
+ * @see XXH3_createState(), XXH3_freeState().
1537
+ * @see XXH32_state_s, XXH64_state_s
1538
+ */
1539
+struct XXH3_state_s {
1540
+ XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
1541
+ /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */
1542
+ XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
1543
+ /*!< Used to store a custom secret generated from a seed. */
1544
+ XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
1545
+ /*!< The internal buffer. @see XXH32_state_s::mem32 */
1546
+ XXH32_hash_t bufferedSize;
1547
+ /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */
1548
+ XXH32_hash_t useSeed;
1549
+ /*!< Reserved field. Needed for padding on 64-bit. */
1550
+ size_t nbStripesSoFar;
1551
+ /*!< Number or stripes processed. */
1552
+ XXH64_hash_t totalLen;
1553
+ /*!< Total length hashed. 64-bit even on 32-bit targets. */
1554
+ size_t nbStripesPerBlock;
1555
+ /*!< Number of stripes per block. */
1556
+ size_t secretLimit;
1557
+ /*!< Size of @ref customSecret or @ref extSecret */
1558
+ XXH64_hash_t seed;
1559
+ /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */
1560
+ XXH64_hash_t reserved64;
1561
+ /*!< Reserved field. */
1562
+ const unsigned char* extSecret;
1563
+ /*!< Reference to an external secret for the _withSecret variants, NULL
1564
+ * for other variants. */
1565
+ /* note: there may be some padding at the end due to alignment on 64 bytes */
1566
+}; /* typedef'd to XXH3_state_t */
1567
+
1568
+#undef XXH_ALIGN_MEMBER
1569
+
1570
+/*!
1571
+ * @brief Initializes a stack-allocated `XXH3_state_s`.
1572
+ *
1573
+ * When the @ref XXH3_state_t structure is merely emplaced on stack,
1574
+ * it should be initialized with XXH3_INITSTATE() or a memset()
1575
+ * in case its first reset uses XXH3_NNbits_reset_withSeed().
1576
+ * This init can be omitted if the first reset uses default or _withSecret mode.
1577
+ * This operation isn't necessary when the state is created with XXH3_createState().
1578
+ * Note that this doesn't prepare the state for a streaming operation,
1579
+ * it's still necessary to use XXH3_NNbits_reset*() afterwards.
1580
+ */
1581
+#define XXH3_INITSTATE(XXH3_state_ptr) \
1582
+ do { \
1583
+ XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \
1584
+ tmp_xxh3_state_ptr->seed = 0; \
1585
+ tmp_xxh3_state_ptr->extSecret = NULL; \
1586
+ } while(0)
1587
+
1588
+
1589
+/*!
1590
+ * simple alias to pre-selected XXH3_128bits variant
1591
+ */
1592
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
1593
+
1594
+
1595
+/* === Experimental API === */
1596
+/* Symbols defined below must be considered tied to a specific library version. */
1597
+
1598
+/*!
1599
+ * XXH3_generateSecret():
1600
+ *
1601
+ * Derive a high-entropy secret from any user-defined content, named customSeed.
1602
+ * The generated secret can be used in combination with `*_withSecret()` functions.
1603
+ * The `_withSecret()` variants are useful to provide a higher level of protection
1604
+ * than 64-bit seed, as it becomes much more difficult for an external actor to
1605
+ * guess how to impact the calculation logic.
1606
+ *
1607
+ * The function accepts as input a custom seed of any length and any content,
1608
+ * and derives from it a high-entropy secret of length @p secretSize into an
1609
+ * already allocated buffer @p secretBuffer.
1610
+ *
1611
+ * The generated secret can then be used with any `*_withSecret()` variant.
1612
+ * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(),
1613
+ * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret()
1614
+ * are part of this list. They all accept a `secret` parameter
1615
+ * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN)
1616
+ * _and_ feature very high entropy (consist of random-looking bytes).
1617
+ * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can
1618
+ * be employed to ensure proper quality.
1619
+ *
1620
+ * @p customSeed can be anything. It can have any size, even small ones,
1621
+ * and its content can be anything, even "poor entropy" sources such as a bunch
1622
+ * of zeroes. The resulting `secret` will nonetheless provide all required qualities.
1623
+ *
1624
+ * @pre
1625
+ * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN
1626
+ * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
1627
+ *
1628
+ * Example code:
1629
+ * @code{.c}
1630
+ * #include <stdio.h>
1631
+ * #include <stdlib.h>
1632
+ * #include <string.h>
1633
+ * #define XXH_STATIC_LINKING_ONLY // expose unstable API
1634
+ * #include "xxhash.h"
1635
+ * // Hashes argv[2] using the entropy from argv[1].
1636
+ * int main(int argc, char* argv[])
1637
+ * {
1638
+ * char secret[XXH3_SECRET_SIZE_MIN];
1639
+ * if (argv != 3) { return 1; }
1640
+ * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1]));
1641
+ * XXH64_hash_t h = XXH3_64bits_withSecret(
1642
+ * argv[2], strlen(argv[2]),
1643
+ * secret, sizeof(secret)
1644
+ * );
1645
+ * printf("%016llx\n", (unsigned long long) h);
1646
+ * }
1647
+ * @endcode
1648
+ */
1649
+XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize);
1650
+
1651
+/*!
1652
+ * @brief Generate the same secret as the _withSeed() variants.
1653
+ *
1654
+ * The generated secret can be used in combination with
1655
+ *`*_withSecret()` and `_withSecretandSeed()` variants.
1656
+ *
1657
+ * Example C++ `std::string` hash class:
1658
+ * @code{.cpp}
1659
+ * #include <string>
1660
+ * #define XXH_STATIC_LINKING_ONLY // expose unstable API
1661
+ * #include "xxhash.h"
1662
+ * // Slow, seeds each time
1663
+ * class HashSlow {
1664
+ * XXH64_hash_t seed;
1665
+ * public:
1666
+ * HashSlow(XXH64_hash_t s) : seed{s} {}
1667
+ * size_t operator()(const std::string& x) const {
1668
+ * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)};
1669
+ * }
1670
+ * };
1671
+ * // Fast, caches the seeded secret for future uses.
1672
+ * class HashFast {
1673
+ * unsigned char secret[XXH3_SECRET_SIZE_MIN];
1674
+ * public:
1675
+ * HashFast(XXH64_hash_t s) {
1676
+ * XXH3_generateSecret_fromSeed(secret, seed);
1677
+ * }
1678
+ * size_t operator()(const std::string& x) const {
1679
+ * return size_t{
1680
+ * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret))
1681
+ * };
1682
+ * }
1683
+ * };
1684
+ * @endcode
1685
+ * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes
1686
+ * @param seed The seed to seed the state.
1687
+ */
1688
+XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed);
1689
+
1690
+/*!
1691
+ * These variants generate hash values using either
1692
+ * @p seed for "short" keys (< XXH3_MIDSIZE_MAX = 240 bytes)
1693
+ * or @p secret for "large" keys (>= XXH3_MIDSIZE_MAX).
1694
+ *
1695
+ * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
1696
+ * `_withSeed()` has to generate the secret on the fly for "large" keys.
1697
+ * It's fast, but can be perceptible for "not so large" keys (< 1 KB).
1698
+ * `_withSecret()` has to generate the masks on the fly for "small" keys,
1699
+ * which requires more instructions than _withSeed() variants.
1700
+ * Therefore, _withSecretandSeed variant combines the best of both worlds.
1701
+ *
1702
+ * When @p secret has been generated by XXH3_generateSecret_fromSeed(),
1703
+ * this variant produces *exactly* the same results as `_withSeed()` variant,
1704
+ * hence offering only a pure speed benefit on "large" input,
1705
+ * by skipping the need to regenerate the secret for every large input.
1706
+ *
1707
+ * Another usage scenario is to hash the secret to a 64-bit hash value,
1708
+ * for example with XXH3_64bits(), which then becomes the seed,
1709
+ * and then employ both the seed and the secret in _withSecretandSeed().
1710
+ * On top of speed, an added benefit is that each bit in the secret
1711
+ * has a 50% chance to swap each bit in the output, via its impact to the seed.
1712
+ *
1713
+ * This is not guaranteed when using the secret directly in "small data" scenarios,
1714
+ * because only portions of the secret are employed for small data.
1715
+ */
1716
+XXH_PUBLIC_API XXH_PUREF XXH64_hash_t
1717
+XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len,
1718
+ XXH_NOESCAPE const void* secret, size_t secretSize,
1719
+ XXH64_hash_t seed);
1720
+/*! @copydoc XXH3_64bits_withSecretandSeed() */
1721
+XXH_PUBLIC_API XXH_PUREF XXH128_hash_t
1722
+XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length,
1723
+ XXH_NOESCAPE const void* secret, size_t secretSize,
1724
+ XXH64_hash_t seed64);
1725
+#ifndef XXH_NO_STREAM
1726
+/*! @copydoc XXH3_64bits_withSecretandSeed() */
1727
+XXH_PUBLIC_API XXH_errorcode
1728
+XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
1729
+ XXH_NOESCAPE const void* secret, size_t secretSize,
1730
+ XXH64_hash_t seed64);
1731
+/*! @copydoc XXH3_64bits_withSecretandSeed() */
1732
+XXH_PUBLIC_API XXH_errorcode
1733
+XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
1734
+ XXH_NOESCAPE const void* secret, size_t secretSize,
1735
+ XXH64_hash_t seed64);
1736
+#endif /* !XXH_NO_STREAM */
1737
+
1738
+#endif /* !XXH_NO_XXH3 */
1739
+#endif /* XXH_NO_LONG_LONG */
1740
+#if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
1741
+# define XXH_IMPLEMENTATION
1742
+#endif
1743
+
1744
+#endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
1745
+
1746
+
1747
+/* ======================================================================== */
1748
+/* ======================================================================== */
1749
+/* ======================================================================== */
1750
+
1751
+
1752
+/*-**********************************************************************
1753
+ * xxHash implementation
1754
+ *-**********************************************************************
1755
+ * xxHash's implementation used to be hosted inside xxhash.c.
1756
+ *
1757
+ * However, inlining requires implementation to be visible to the compiler,
1758
+ * hence be included alongside the header.
1759
+ * Previously, implementation was hosted inside xxhash.c,
1760
+ * which was then #included when inlining was activated.
1761
+ * This construction created issues with a few build and install systems,
1762
+ * as it required xxhash.c to be stored in /include directory.
1763
+ *
1764
+ * xxHash implementation is now directly integrated within xxhash.h.
1765
+ * As a consequence, xxhash.c is no longer needed in /include.
1766
+ *
1767
+ * xxhash.c is still available and is still useful.
1768
+ * In a "normal" setup, when xxhash is not inlined,
1769
+ * xxhash.h only exposes the prototypes and public symbols,
1770
+ * while xxhash.c can be built into an object file xxhash.o
1771
+ * which can then be linked into the final binary.
1772
+ ************************************************************************/
1773
+
1774
+#if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \
1775
+ || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
1776
+# define XXH_IMPLEM_13a8737387
1777
+
1778
+/* *************************************
1779
+* Tuning parameters
1780
+***************************************/
1781
+
1782
+/*!
1783
+ * @defgroup tuning Tuning parameters
1784
+ * @{
1785
+ *
1786
+ * Various macros to control xxHash's behavior.
1787
+ */
1788
+#ifdef XXH_DOXYGEN
1789
+/*!
1790
+ * @brief Define this to disable 64-bit code.
1791
+ *
1792
+ * Useful if only using the @ref XXH32_family and you have a strict C90 compiler.
1793
+ */
1794
+# define XXH_NO_LONG_LONG
1795
+# undef XXH_NO_LONG_LONG /* don't actually */
1796
+/*!
1797
+ * @brief Controls how unaligned memory is accessed.
1798
+ *
1799
+ * By default, access to unaligned memory is controlled by `memcpy()`, which is
1800
+ * safe and portable.
1801
+ *
1802
+ * Unfortunately, on some target/compiler combinations, the generated assembly
1803
+ * is sub-optimal.
1804
+ *
1805
+ * The below switch allow selection of a different access method
1806
+ * in the search for improved performance.
1807
+ *
1808
+ * @par Possible options:
1809
+ *
1810
+ * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy`
1811
+ * @par
1812
+ * Use `memcpy()`. Safe and portable. Note that most modern compilers will
1813
+ * eliminate the function call and treat it as an unaligned access.
1814
+ *
1815
+ * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))`
1816
+ * @par
1817
+ * Depends on compiler extensions and is therefore not portable.
1818
+ * This method is safe _if_ your compiler supports it,
1819
+ * and *generally* as fast or faster than `memcpy`.
1820
+ *
1821
+ * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast
1822
+ * @par
1823
+ * Casts directly and dereferences. This method doesn't depend on the
1824
+ * compiler, but it violates the C standard as it directly dereferences an
1825
+ * unaligned pointer. It can generate buggy code on targets which do not
1826
+ * support unaligned memory accesses, but in some circumstances, it's the
1827
+ * only known way to get the most performance.
1828
+ *
1829
+ * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift
1830
+ * @par
1831
+ * Also portable. This can generate the best code on old compilers which don't
1832
+ * inline small `memcpy()` calls, and it might also be faster on big-endian
1833
+ * systems which lack a native byteswap instruction. However, some compilers
1834
+ * will emit literal byteshifts even if the target supports unaligned access.
1835
+ *
1836
+ *
1837
+ * @warning
1838
+ * Methods 1 and 2 rely on implementation-defined behavior. Use these with
1839
+ * care, as what works on one compiler/platform/optimization level may cause
1840
+ * another to read garbage data or even crash.
1841
+ *
1842
+ * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
1843
+ *
1844
+ * Prefer these methods in priority order (0 > 3 > 1 > 2)
1845
+ */
1846
+# define XXH_FORCE_MEMORY_ACCESS 0
1847
+
1848
+/*!
1849
+ * @def XXH_SIZE_OPT
1850
+ * @brief Controls how much xxHash optimizes for size.
1851
+ *
1852
+ * xxHash, when compiled, tends to result in a rather large binary size. This
1853
+ * is mostly due to heavy usage to forced inlining and constant folding of the
1854
+ * @ref XXH3_family to increase performance.
1855
+ *
1856
+ * However, some developers prefer size over speed. This option can
1857
+ * significantly reduce the size of the generated code. When using the `-Os`
1858
+ * or `-Oz` options on GCC or Clang, this is defined to 1 by default,
1859
+ * otherwise it is defined to 0.
1860
+ *
1861
+ * Most of these size optimizations can be controlled manually.
1862
+ *
1863
+ * This is a number from 0-2.
1864
+ * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed
1865
+ * comes first.
1866
+ * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more
1867
+ * conservative and disables hacks that increase code size. It implies the
1868
+ * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0,
1869
+ * and @ref XXH3_NEON_LANES == 8 if they are not already defined.
1870
+ * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible.
1871
+ * Performance may cry. For example, the single shot functions just use the
1872
+ * streaming API.
1873
+ */
1874
+# define XXH_SIZE_OPT 0
1875
+
1876
+/*!
1877
+ * @def XXH_FORCE_ALIGN_CHECK
1878
+ * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32()
1879
+ * and XXH64() only).
1880
+ *
1881
+ * This is an important performance trick for architectures without decent
1882
+ * unaligned memory access performance.
1883
+ *
1884
+ * It checks for input alignment, and when conditions are met, uses a "fast
1885
+ * path" employing direct 32-bit/64-bit reads, resulting in _dramatically
1886
+ * faster_ read speed.
1887
+ *
1888
+ * The check costs one initial branch per hash, which is generally negligible,
1889
+ * but not zero.
1890
+ *
1891
+ * Moreover, it's not useful to generate an additional code path if memory
1892
+ * access uses the same instruction for both aligned and unaligned
1893
+ * addresses (e.g. x86 and aarch64).
1894
+ *
1895
+ * In these cases, the alignment check can be removed by setting this macro to 0.
1896
+ * Then the code will always use unaligned memory access.
1897
+ * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips
1898
+ * which are platforms known to offer good unaligned memory accesses performance.
1899
+ *
1900
+ * It is also disabled by default when @ref XXH_SIZE_OPT >= 1.
1901
+ *
1902
+ * This option does not affect XXH3 (only XXH32 and XXH64).
1903
+ */
1904
+# define XXH_FORCE_ALIGN_CHECK 0
1905
+
1906
+/*!
1907
+ * @def XXH_NO_INLINE_HINTS
1908
+ * @brief When non-zero, sets all functions to `static`.
1909
+ *
1910
+ * By default, xxHash tries to force the compiler to inline almost all internal
1911
+ * functions.
1912
+ *
1913
+ * This can usually improve performance due to reduced jumping and improved
1914
+ * constant folding, but significantly increases the size of the binary which
1915
+ * might not be favorable.
1916
+ *
1917
+ * Additionally, sometimes the forced inlining can be detrimental to performance,
1918
+ * depending on the architecture.
1919
+ *
1920
+ * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
1921
+ * compiler full control on whether to inline or not.
1922
+ *
1923
+ * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if
1924
+ * @ref XXH_SIZE_OPT >= 1, this will automatically be defined.
1925
+ */
1926
+# define XXH_NO_INLINE_HINTS 0
1927
+
1928
+/*!
1929
+ * @def XXH3_INLINE_SECRET
1930
+ * @brief Determines whether to inline the XXH3 withSecret code.
1931
+ *
1932
+ * When the secret size is known, the compiler can improve the performance
1933
+ * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret().
1934
+ *
1935
+ * However, if the secret size is not known, it doesn't have any benefit. This
1936
+ * happens when xxHash is compiled into a global symbol. Therefore, if
1937
+ * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0.
1938
+ *
1939
+ * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers
1940
+ * that are *sometimes* force inline on -Og, and it is impossible to automatically
1941
+ * detect this optimization level.
1942
+ */
1943
+# define XXH3_INLINE_SECRET 0
1944
+
1945
+/*!
1946
+ * @def XXH32_ENDJMP
1947
+ * @brief Whether to use a jump for `XXH32_finalize`.
1948
+ *
1949
+ * For performance, `XXH32_finalize` uses multiple branches in the finalizer.
1950
+ * This is generally preferable for performance,
1951
+ * but depending on exact architecture, a jmp may be preferable.
1952
+ *
1953
+ * This setting is only possibly making a difference for very small inputs.
1954
+ */
1955
+# define XXH32_ENDJMP 0
1956
+
1957
+/*!
1958
+ * @internal
1959
+ * @brief Redefines old internal names.
1960
+ *
1961
+ * For compatibility with code that uses xxHash's internals before the names
1962
+ * were changed to improve namespacing. There is no other reason to use this.
1963
+ */
1964
+# define XXH_OLD_NAMES
1965
+# undef XXH_OLD_NAMES /* don't actually use, it is ugly. */
1966
+
1967
+/*!
1968
+ * @def XXH_NO_STREAM
1969
+ * @brief Disables the streaming API.
1970
+ *
1971
+ * When xxHash is not inlined and the streaming functions are not used, disabling
1972
+ * the streaming functions can improve code size significantly, especially with
1973
+ * the @ref XXH3_family which tends to make constant folded copies of itself.
1974
+ */
1975
+# define XXH_NO_STREAM
1976
+# undef XXH_NO_STREAM /* don't actually */
1977
+#endif /* XXH_DOXYGEN */
1978
+/*!
1979
+ * @}
1980
+ */
1981
+
1982
+#ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
1983
+ /* prefer __packed__ structures (method 1) for GCC
1984
+ * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy
1985
+ * which for some reason does unaligned loads. */
1986
+# if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED))
1987
+# define XXH_FORCE_MEMORY_ACCESS 1
1988
+# endif
1989
+#endif
1990
+
1991
+#ifndef XXH_SIZE_OPT
1992
+ /* default to 1 for -Os or -Oz */
1993
+# if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__)
1994
+# define XXH_SIZE_OPT 1
1995
+# else
1996
+# define XXH_SIZE_OPT 0
1997
+# endif
1998
+#endif
1999
+
2000
+#ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
2001
+ /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */
2002
+# if XXH_SIZE_OPT >= 1 || \
2003
+ defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \
2004
+ || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */
2005
+# define XXH_FORCE_ALIGN_CHECK 0
2006
+# else
2007
+# define XXH_FORCE_ALIGN_CHECK 1
2008
+# endif
2009
+#endif
2010
+
2011
+#ifndef XXH_NO_INLINE_HINTS
2012
+# if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */
2013
+# define XXH_NO_INLINE_HINTS 1
2014
+# else
2015
+# define XXH_NO_INLINE_HINTS 0
2016
+# endif
2017
+#endif
2018
+
2019
+#ifndef XXH3_INLINE_SECRET
2020
+# if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \
2021
+ || !defined(XXH_INLINE_ALL)
2022
+# define XXH3_INLINE_SECRET 0
2023
+# else
2024
+# define XXH3_INLINE_SECRET 1
2025
+# endif
2026
+#endif
2027
+
2028
+#ifndef XXH32_ENDJMP
2029
+/* generally preferable for performance */
2030
+# define XXH32_ENDJMP 0
2031
+#endif
2032
+
2033
+/*!
2034
+ * @defgroup impl Implementation
2035
+ * @{
2036
+ */
2037
+
2038
+
2039
+/* *************************************
2040
+* Includes & Memory related functions
2041
+***************************************/
2042
+#if defined(XXH_NO_STREAM)
2043
+/* nothing */
2044
+#elif defined(XXH_NO_STDLIB)
2045
+
2046
+/* When requesting to disable any mention of stdlib,
2047
+ * the library loses the ability to invoked malloc / free.
2048
+ * In practice, it means that functions like `XXH*_createState()`
2049
+ * will always fail, and return NULL.
2050
+ * This flag is useful in situations where
2051
+ * xxhash.h is integrated into some kernel, embedded or limited environment
2052
+ * without access to dynamic allocation.
2053
+ */
2054
+
2055
+static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; }
2056
+static void XXH_free(void* p) { (void)p; }
2057
+
2058
+#else
2059
+
2060
+/*
2061
+ * Modify the local functions below should you wish to use
2062
+ * different memory routines for malloc() and free()
2063
+ */
2064
+#include <stdlib.h>
2065
+
2066
+/*!
2067
+ * @internal
2068
+ * @brief Modify this function to use a different routine than malloc().
2069
+ */
2070
+static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); }
2071
+
2072
+/*!
2073
+ * @internal
2074
+ * @brief Modify this function to use a different routine than free().
2075
+ */
2076
+static void XXH_free(void* p) { free(p); }
2077
+
2078
+#endif /* XXH_NO_STDLIB */
2079
+
2080
+#include <string.h>
2081
+
2082
+/*!
2083
+ * @internal
2084
+ * @brief Modify this function to use a different routine than memcpy().
2085
+ */
2086
+static void* XXH_memcpy(void* dest, const void* src, size_t size)
2087
+{
2088
+ return memcpy(dest,src,size);
2089
+}
2090
+
2091
+#include <limits.h> /* ULLONG_MAX */
2092
+
2093
+
2094
+/* *************************************
2095
+* Compiler Specific Options
2096
+***************************************/
2097
+#ifdef _MSC_VER /* Visual Studio warning fix */
2098
+# pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
2099
+#endif
2100
+
2101
+#if XXH_NO_INLINE_HINTS /* disable inlining hints */
2102
+# if defined(__GNUC__) || defined(__clang__)
2103
+# define XXH_FORCE_INLINE static __attribute__((unused))
2104
+# else
2105
+# define XXH_FORCE_INLINE static
2106
+# endif
2107
+# define XXH_NO_INLINE static
2108
+/* enable inlining hints */
2109
+#elif defined(__GNUC__) || defined(__clang__)
2110
+# define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused))
2111
+# define XXH_NO_INLINE static __attribute__((noinline))
2112
+#elif defined(_MSC_VER) /* Visual Studio */
2113
+# define XXH_FORCE_INLINE static __forceinline
2114
+# define XXH_NO_INLINE static __declspec(noinline)
2115
+#elif defined (__cplusplus) \
2116
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */
2117
+# define XXH_FORCE_INLINE static inline
2118
+# define XXH_NO_INLINE static
2119
+#else
2120
+# define XXH_FORCE_INLINE static
2121
+# define XXH_NO_INLINE static
2122
+#endif
2123
+
2124
+#if XXH3_INLINE_SECRET
2125
+# define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE
2126
+#else
2127
+# define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE
2128
+#endif
2129
+
2130
+
2131
+/* *************************************
2132
+* Debug
2133
+***************************************/
2134
+/*!
2135
+ * @ingroup tuning
2136
+ * @def XXH_DEBUGLEVEL
2137
+ * @brief Sets the debugging level.
2138
+ *
2139
+ * XXH_DEBUGLEVEL is expected to be defined externally, typically via the
2140
+ * compiler's command line options. The value must be a number.
2141
+ */
2142
+#ifndef XXH_DEBUGLEVEL
2143
+# ifdef DEBUGLEVEL /* backwards compat */
2144
+# define XXH_DEBUGLEVEL DEBUGLEVEL
2145
+# else
2146
+# define XXH_DEBUGLEVEL 0
2147
+# endif
2148
+#endif
2149
+
2150
+#if (XXH_DEBUGLEVEL>=1)
2151
+# include <assert.h> /* note: can still be disabled with NDEBUG */
2152
+# define XXH_ASSERT(c) assert(c)
2153
+#else
2154
+# if defined(__INTEL_COMPILER)
2155
+# define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c))
2156
+# else
2157
+# define XXH_ASSERT(c) XXH_ASSUME(c)
2158
+# endif
2159
+#endif
2160
+
2161
+/* note: use after variable declarations */
2162
+#ifndef XXH_STATIC_ASSERT
2163
+# if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
2164
+# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0)
2165
+# elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */
2166
+# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
2167
+# else
2168
+# define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
2169
+# endif
2170
+# define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c)
2171
+#endif
2172
+
2173
+/*!
2174
+ * @internal
2175
+ * @def XXH_COMPILER_GUARD(var)
2176
+ * @brief Used to prevent unwanted optimizations for @p var.
2177
+ *
2178
+ * It uses an empty GCC inline assembly statement with a register constraint
2179
+ * which forces @p var into a general purpose register (eg eax, ebx, ecx
2180
+ * on x86) and marks it as modified.
2181
+ *
2182
+ * This is used in a few places to avoid unwanted autovectorization (e.g.
2183
+ * XXH32_round()). All vectorization we want is explicit via intrinsics,
2184
+ * and _usually_ isn't wanted elsewhere.
2185
+ *
2186
+ * We also use it to prevent unwanted constant folding for AArch64 in
2187
+ * XXH3_initCustomSecret_scalar().
2188
+ */
2189
+#if defined(__GNUC__) || defined(__clang__)
2190
+# define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var))
2191
+#else
2192
+# define XXH_COMPILER_GUARD(var) ((void)0)
2193
+#endif
2194
+
2195
+/* Specifically for NEON vectors which use the "w" constraint, on
2196
+ * Clang. */
2197
+#if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__)
2198
+# define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var))
2199
+#else
2200
+# define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0)
2201
+#endif
2202
+
2203
+/* *************************************
2204
+* Basic Types
2205
+***************************************/
2206
+#if !defined (__VMS) \
2207
+ && (defined (__cplusplus) \
2208
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
2209
+# include <stdint.h>
2210
+ typedef uint8_t xxh_u8;
2211
+#else
2212
+ typedef unsigned char xxh_u8;
2213
+#endif
2214
+typedef XXH32_hash_t xxh_u32;
2215
+
2216
+#ifdef XXH_OLD_NAMES
2217
+# warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly"
2218
+# define BYTE xxh_u8
2219
+# define U8 xxh_u8
2220
+# define U32 xxh_u32
2221
+#endif
2222
+
2223
+/* *** Memory access *** */
2224
+
2225
+/*!
2226
+ * @internal
2227
+ * @fn xxh_u32 XXH_read32(const void* ptr)
2228
+ * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness.
2229
+ *
2230
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2231
+ *
2232
+ * @param ptr The pointer to read from.
2233
+ * @return The 32-bit native endian integer from the bytes at @p ptr.
2234
+ */
2235
+
2236
+/*!
2237
+ * @internal
2238
+ * @fn xxh_u32 XXH_readLE32(const void* ptr)
2239
+ * @brief Reads an unaligned 32-bit little endian integer from @p ptr.
2240
+ *
2241
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2242
+ *
2243
+ * @param ptr The pointer to read from.
2244
+ * @return The 32-bit little endian integer from the bytes at @p ptr.
2245
+ */
2246
+
2247
+/*!
2248
+ * @internal
2249
+ * @fn xxh_u32 XXH_readBE32(const void* ptr)
2250
+ * @brief Reads an unaligned 32-bit big endian integer from @p ptr.
2251
+ *
2252
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2253
+ *
2254
+ * @param ptr The pointer to read from.
2255
+ * @return The 32-bit big endian integer from the bytes at @p ptr.
2256
+ */
2257
+
2258
+/*!
2259
+ * @internal
2260
+ * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align)
2261
+ * @brief Like @ref XXH_readLE32(), but has an option for aligned reads.
2262
+ *
2263
+ * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2264
+ * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is
2265
+ * always @ref XXH_alignment::XXH_unaligned.
2266
+ *
2267
+ * @param ptr The pointer to read from.
2268
+ * @param align Whether @p ptr is aligned.
2269
+ * @pre
2270
+ * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte
2271
+ * aligned.
2272
+ * @return The 32-bit little endian integer from the bytes at @p ptr.
2273
+ */
2274
+
2275
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
2276
+/*
2277
+ * Manual byteshift. Best for old compilers which don't inline memcpy.
2278
+ * We actually directly use XXH_readLE32 and XXH_readBE32.
2279
+ */
2280
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
2281
+
2282
+/*
2283
+ * Force direct memory access. Only works on CPU which support unaligned memory
2284
+ * access in hardware.
2285
+ */
2286
+static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; }
2287
+
2288
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
2289
+
2290
+/*
2291
+ * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
2292
+ * documentation claimed that it only increased the alignment, but actually it
2293
+ * can decrease it on gcc, clang, and icc:
2294
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
2295
+ * https://gcc.godbolt.org/z/xYez1j67Y.
2296
+ */
2297
+#ifdef XXH_OLD_NAMES
2298
+typedef union { xxh_u32 u32; } __attribute__((packed)) unalign;
2299
+#endif
2300
+static xxh_u32 XXH_read32(const void* ptr)
2301
+{
2302
+ typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32;
2303
+ return *((const xxh_unalign32*)ptr);
2304
+}
2305
+
2306
+#else
2307
+
2308
+/*
2309
+ * Portable and safe solution. Generally efficient.
2310
+ * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
2311
+ */
2312
+static xxh_u32 XXH_read32(const void* memPtr)
2313
+{
2314
+ xxh_u32 val;
2315
+ XXH_memcpy(&val, memPtr, sizeof(val));
2316
+ return val;
2317
+}
2318
+
2319
+#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
2320
+
2321
+
2322
+/* *** Endianness *** */
2323
+
2324
+/*!
2325
+ * @ingroup tuning
2326
+ * @def XXH_CPU_LITTLE_ENDIAN
2327
+ * @brief Whether the target is little endian.
2328
+ *
2329
+ * Defined to 1 if the target is little endian, or 0 if it is big endian.
2330
+ * It can be defined externally, for example on the compiler command line.
2331
+ *
2332
+ * If it is not defined,
2333
+ * a runtime check (which is usually constant folded) is used instead.
2334
+ *
2335
+ * @note
2336
+ * This is not necessarily defined to an integer constant.
2337
+ *
2338
+ * @see XXH_isLittleEndian() for the runtime check.
2339
+ */
2340
+#ifndef XXH_CPU_LITTLE_ENDIAN
2341
+/*
2342
+ * Try to detect endianness automatically, to avoid the nonstandard behavior
2343
+ * in `XXH_isLittleEndian()`
2344
+ */
2345
+# if defined(_WIN32) /* Windows is always little endian */ \
2346
+ || defined(__LITTLE_ENDIAN__) \
2347
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
2348
+# define XXH_CPU_LITTLE_ENDIAN 1
2349
+# elif defined(__BIG_ENDIAN__) \
2350
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
2351
+# define XXH_CPU_LITTLE_ENDIAN 0
2352
+# else
2353
+/*!
2354
+ * @internal
2355
+ * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN.
2356
+ *
2357
+ * Most compilers will constant fold this.
2358
+ */
2359
+static int XXH_isLittleEndian(void)
2360
+{
2361
+ /*
2362
+ * Portable and well-defined behavior.
2363
+ * Don't use static: it is detrimental to performance.
2364
+ */
2365
+ const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
2366
+ return one.c[0];
2367
+}
2368
+# define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian()
2369
+# endif
2370
+#endif
2371
+
2372
+
2373
+
2374
+
2375
+/* ****************************************
2376
+* Compiler-specific Functions and Macros
2377
+******************************************/
2378
+#define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
2379
+
2380
+#ifdef __has_builtin
2381
+# define XXH_HAS_BUILTIN(x) __has_builtin(x)
2382
+#else
2383
+# define XXH_HAS_BUILTIN(x) 0
2384
+#endif
2385
+
2386
+
2387
+
2388
+/*
2389
+ * C23 and future versions have standard "unreachable()".
2390
+ * Once it has been implemented reliably we can add it as an
2391
+ * additional case:
2392
+ *
2393
+ * ```
2394
+ * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN)
2395
+ * # include <stddef.h>
2396
+ * # ifdef unreachable
2397
+ * # define XXH_UNREACHABLE() unreachable()
2398
+ * # endif
2399
+ * #endif
2400
+ * ```
2401
+ *
2402
+ * Note C++23 also has std::unreachable() which can be detected
2403
+ * as follows:
2404
+ * ```
2405
+ * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L)
2406
+ * # include <utility>
2407
+ * # define XXH_UNREACHABLE() std::unreachable()
2408
+ * #endif
2409
+ * ```
2410
+ * NB: `__cpp_lib_unreachable` is defined in the `<version>` header.
2411
+ * We don't use that as including `<utility>` in `extern "C"` blocks
2412
+ * doesn't work on GCC12
2413
+ */
2414
+
2415
+#if XXH_HAS_BUILTIN(__builtin_unreachable)
2416
+# define XXH_UNREACHABLE() __builtin_unreachable()
2417
+
2418
+#elif defined(_MSC_VER)
2419
+# define XXH_UNREACHABLE() __assume(0)
2420
+
2421
+#else
2422
+# define XXH_UNREACHABLE()
2423
+#endif
2424
+
2425
+#if XXH_HAS_BUILTIN(__builtin_assume)
2426
+# define XXH_ASSUME(c) __builtin_assume(c)
2427
+#else
2428
+# define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); }
2429
+#endif
2430
+
2431
+/*!
2432
+ * @internal
2433
+ * @def XXH_rotl32(x,r)
2434
+ * @brief 32-bit rotate left.
2435
+ *
2436
+ * @param x The 32-bit integer to be rotated.
2437
+ * @param r The number of bits to rotate.
2438
+ * @pre
2439
+ * @p r > 0 && @p r < 32
2440
+ * @note
2441
+ * @p x and @p r may be evaluated multiple times.
2442
+ * @return The rotated result.
2443
+ */
2444
+#if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
2445
+ && XXH_HAS_BUILTIN(__builtin_rotateleft64)
2446
+# define XXH_rotl32 __builtin_rotateleft32
2447
+# define XXH_rotl64 __builtin_rotateleft64
2448
+/* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
2449
+#elif defined(_MSC_VER)
2450
+# define XXH_rotl32(x,r) _rotl(x,r)
2451
+# define XXH_rotl64(x,r) _rotl64(x,r)
2452
+#else
2453
+# define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
2454
+# define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
2455
+#endif
2456
+
2457
+/*!
2458
+ * @internal
2459
+ * @fn xxh_u32 XXH_swap32(xxh_u32 x)
2460
+ * @brief A 32-bit byteswap.
2461
+ *
2462
+ * @param x The 32-bit integer to byteswap.
2463
+ * @return @p x, byteswapped.
2464
+ */
2465
+#if defined(_MSC_VER) /* Visual Studio */
2466
+# define XXH_swap32 _byteswap_ulong
2467
+#elif XXH_GCC_VERSION >= 403
2468
+# define XXH_swap32 __builtin_bswap32
2469
+#else
2470
+static xxh_u32 XXH_swap32 (xxh_u32 x)
2471
+{
2472
+ return ((x << 24) & 0xff000000 ) |
2473
+ ((x << 8) & 0x00ff0000 ) |
2474
+ ((x >> 8) & 0x0000ff00 ) |
2475
+ ((x >> 24) & 0x000000ff );
2476
+}
2477
+#endif
2478
+
2479
+
2480
+/* ***************************
2481
+* Memory reads
2482
+*****************************/
2483
+
2484
+/*!
2485
+ * @internal
2486
+ * @brief Enum to indicate whether a pointer is aligned.
2487
+ */
2488
+typedef enum {
2489
+ XXH_aligned, /*!< Aligned */
2490
+ XXH_unaligned /*!< Possibly unaligned */
2491
+} XXH_alignment;
2492
+
2493
+/*
2494
+ * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
2495
+ *
2496
+ * This is ideal for older compilers which don't inline memcpy.
2497
+ */
2498
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
2499
+
2500
+XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
2501
+{
2502
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
2503
+ return bytePtr[0]
2504
+ | ((xxh_u32)bytePtr[1] << 8)
2505
+ | ((xxh_u32)bytePtr[2] << 16)
2506
+ | ((xxh_u32)bytePtr[3] << 24);
2507
+}
2508
+
2509
+XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
2510
+{
2511
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
2512
+ return bytePtr[3]
2513
+ | ((xxh_u32)bytePtr[2] << 8)
2514
+ | ((xxh_u32)bytePtr[1] << 16)
2515
+ | ((xxh_u32)bytePtr[0] << 24);
2516
+}
2517
+
2518
+#else
2519
+XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
2520
+{
2521
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
2522
+}
2523
+
2524
+static xxh_u32 XXH_readBE32(const void* ptr)
2525
+{
2526
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
2527
+}
2528
+#endif
2529
+
2530
+XXH_FORCE_INLINE xxh_u32
2531
+XXH_readLE32_align(const void* ptr, XXH_alignment align)
2532
+{
2533
+ if (align==XXH_unaligned) {
2534
+ return XXH_readLE32(ptr);
2535
+ } else {
2536
+ return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr);
2537
+ }
2538
+}
2539
+
2540
+
2541
+/* *************************************
2542
+* Misc
2543
+***************************************/
2544
+/*! @ingroup public */
2545
+XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
2546
+
2547
+
2548
+/* *******************************************************************
2549
+* 32-bit hash functions
2550
+*********************************************************************/
2551
+/*!
2552
+ * @}
2553
+ * @defgroup XXH32_impl XXH32 implementation
2554
+ * @ingroup impl
2555
+ *
2556
+ * Details on the XXH32 implementation.
2557
+ * @{
2558
+ */
2559
+ /* #define instead of static const, to be used as initializers */
2560
+#define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */
2561
+#define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */
2562
+#define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */
2563
+#define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */
2564
+#define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */
2565
+
2566
+#ifdef XXH_OLD_NAMES
2567
+# define PRIME32_1 XXH_PRIME32_1
2568
+# define PRIME32_2 XXH_PRIME32_2
2569
+# define PRIME32_3 XXH_PRIME32_3
2570
+# define PRIME32_4 XXH_PRIME32_4
2571
+# define PRIME32_5 XXH_PRIME32_5
2572
+#endif
2573
+
2574
+/*!
2575
+ * @internal
2576
+ * @brief Normal stripe processing routine.
2577
+ *
2578
+ * This shuffles the bits so that any bit from @p input impacts several bits in
2579
+ * @p acc.
2580
+ *
2581
+ * @param acc The accumulator lane.
2582
+ * @param input The stripe of input to mix.
2583
+ * @return The mixed accumulator lane.
2584
+ */
2585
+static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
2586
+{
2587
+ acc += input * XXH_PRIME32_2;
2588
+ acc = XXH_rotl32(acc, 13);
2589
+ acc *= XXH_PRIME32_1;
2590
+#if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
2591
+ /*
2592
+ * UGLY HACK:
2593
+ * A compiler fence is the only thing that prevents GCC and Clang from
2594
+ * autovectorizing the XXH32 loop (pragmas and attributes don't work for some
2595
+ * reason) without globally disabling SSE4.1.
2596
+ *
2597
+ * The reason we want to avoid vectorization is because despite working on
2598
+ * 4 integers at a time, there are multiple factors slowing XXH32 down on
2599
+ * SSE4:
2600
+ * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
2601
+ * newer chips!) making it slightly slower to multiply four integers at
2602
+ * once compared to four integers independently. Even when pmulld was
2603
+ * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
2604
+ * just to multiply unless doing a long operation.
2605
+ *
2606
+ * - Four instructions are required to rotate,
2607
+ * movqda tmp, v // not required with VEX encoding
2608
+ * pslld tmp, 13 // tmp <<= 13
2609
+ * psrld v, 19 // x >>= 19
2610
+ * por v, tmp // x |= tmp
2611
+ * compared to one for scalar:
2612
+ * roll v, 13 // reliably fast across the board
2613
+ * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason
2614
+ *
2615
+ * - Instruction level parallelism is actually more beneficial here because
2616
+ * the SIMD actually serializes this operation: While v1 is rotating, v2
2617
+ * can load data, while v3 can multiply. SSE forces them to operate
2618
+ * together.
2619
+ *
2620
+ * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing
2621
+ * the loop. NEON is only faster on the A53, and with the newer cores, it is less
2622
+ * than half the speed.
2623
+ *
2624
+ * Additionally, this is used on WASM SIMD128 because it JITs to the same
2625
+ * SIMD instructions and has the same issue.
2626
+ */
2627
+ XXH_COMPILER_GUARD(acc);
2628
+#endif
2629
+ return acc;
2630
+}
2631
+
2632
+/*!
2633
+ * @internal
2634
+ * @brief Mixes all bits to finalize the hash.
2635
+ *
2636
+ * The final mix ensures that all input bits have a chance to impact any bit in
2637
+ * the output digest, resulting in an unbiased distribution.
2638
+ *
2639
+ * @param hash The hash to avalanche.
2640
+ * @return The avalanched hash.
2641
+ */
2642
+static xxh_u32 XXH32_avalanche(xxh_u32 hash)
2643
+{
2644
+ hash ^= hash >> 15;
2645
+ hash *= XXH_PRIME32_2;
2646
+ hash ^= hash >> 13;
2647
+ hash *= XXH_PRIME32_3;
2648
+ hash ^= hash >> 16;
2649
+ return hash;
2650
+}
2651
+
2652
+#define XXH_get32bits(p) XXH_readLE32_align(p, align)
2653
+
2654
+/*!
2655
+ * @internal
2656
+ * @brief Processes the last 0-15 bytes of @p ptr.
2657
+ *
2658
+ * There may be up to 15 bytes remaining to consume from the input.
2659
+ * This final stage will digest them to ensure that all input bytes are present
2660
+ * in the final mix.
2661
+ *
2662
+ * @param hash The hash to finalize.
2663
+ * @param ptr The pointer to the remaining input.
2664
+ * @param len The remaining length, modulo 16.
2665
+ * @param align Whether @p ptr is aligned.
2666
+ * @return The finalized hash.
2667
+ * @see XXH64_finalize().
2668
+ */
2669
+static XXH_PUREF xxh_u32
2670
+XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
2671
+{
2672
+#define XXH_PROCESS1 do { \
2673
+ hash += (*ptr++) * XXH_PRIME32_5; \
2674
+ hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \
2675
+} while (0)
2676
+
2677
+#define XXH_PROCESS4 do { \
2678
+ hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \
2679
+ ptr += 4; \
2680
+ hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \
2681
+} while (0)
2682
+
2683
+ if (ptr==NULL) XXH_ASSERT(len == 0);
2684
+
2685
+ /* Compact rerolled version; generally faster */
2686
+ if (!XXH32_ENDJMP) {
2687
+ len &= 15;
2688
+ while (len >= 4) {
2689
+ XXH_PROCESS4;
2690
+ len -= 4;
2691
+ }
2692
+ while (len > 0) {
2693
+ XXH_PROCESS1;
2694
+ --len;
2695
+ }
2696
+ return XXH32_avalanche(hash);
2697
+ } else {
2698
+ switch(len&15) /* or switch(bEnd - p) */ {
2699
+ case 12: XXH_PROCESS4;
2700
+ XXH_FALLTHROUGH; /* fallthrough */
2701
+ case 8: XXH_PROCESS4;
2702
+ XXH_FALLTHROUGH; /* fallthrough */
2703
+ case 4: XXH_PROCESS4;
2704
+ return XXH32_avalanche(hash);
2705
+
2706
+ case 13: XXH_PROCESS4;
2707
+ XXH_FALLTHROUGH; /* fallthrough */
2708
+ case 9: XXH_PROCESS4;
2709
+ XXH_FALLTHROUGH; /* fallthrough */
2710
+ case 5: XXH_PROCESS4;
2711
+ XXH_PROCESS1;
2712
+ return XXH32_avalanche(hash);
2713
+
2714
+ case 14: XXH_PROCESS4;
2715
+ XXH_FALLTHROUGH; /* fallthrough */
2716
+ case 10: XXH_PROCESS4;
2717
+ XXH_FALLTHROUGH; /* fallthrough */
2718
+ case 6: XXH_PROCESS4;
2719
+ XXH_PROCESS1;
2720
+ XXH_PROCESS1;
2721
+ return XXH32_avalanche(hash);
2722
+
2723
+ case 15: XXH_PROCESS4;
2724
+ XXH_FALLTHROUGH; /* fallthrough */
2725
+ case 11: XXH_PROCESS4;
2726
+ XXH_FALLTHROUGH; /* fallthrough */
2727
+ case 7: XXH_PROCESS4;
2728
+ XXH_FALLTHROUGH; /* fallthrough */
2729
+ case 3: XXH_PROCESS1;
2730
+ XXH_FALLTHROUGH; /* fallthrough */
2731
+ case 2: XXH_PROCESS1;
2732
+ XXH_FALLTHROUGH; /* fallthrough */
2733
+ case 1: XXH_PROCESS1;
2734
+ XXH_FALLTHROUGH; /* fallthrough */
2735
+ case 0: return XXH32_avalanche(hash);
2736
+ }
2737
+ XXH_ASSERT(0);
2738
+ return hash; /* reaching this point is deemed impossible */
2739
+ }
2740
+}
2741
+
2742
+#ifdef XXH_OLD_NAMES
2743
+# define PROCESS1 XXH_PROCESS1
2744
+# define PROCESS4 XXH_PROCESS4
2745
+#else
2746
+# undef XXH_PROCESS1
2747
+# undef XXH_PROCESS4
2748
+#endif
2749
+
2750
+/*!
2751
+ * @internal
2752
+ * @brief The implementation for @ref XXH32().
2753
+ *
2754
+ * @param input , len , seed Directly passed from @ref XXH32().
2755
+ * @param align Whether @p input is aligned.
2756
+ * @return The calculated hash.
2757
+ */
2758
+XXH_FORCE_INLINE XXH_PUREF xxh_u32
2759
+XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
2760
+{
2761
+ xxh_u32 h32;
2762
+
2763
+ if (input==NULL) XXH_ASSERT(len == 0);
2764
+
2765
+ if (len>=16) {
2766
+ const xxh_u8* const bEnd = input + len;
2767
+ const xxh_u8* const limit = bEnd - 15;
2768
+ xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
2769
+ xxh_u32 v2 = seed + XXH_PRIME32_2;
2770
+ xxh_u32 v3 = seed + 0;
2771
+ xxh_u32 v4 = seed - XXH_PRIME32_1;
2772
+
2773
+ do {
2774
+ v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
2775
+ v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
2776
+ v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
2777
+ v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
2778
+ } while (input < limit);
2779
+
2780
+ h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7)
2781
+ + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
2782
+ } else {
2783
+ h32 = seed + XXH_PRIME32_5;
2784
+ }
2785
+
2786
+ h32 += (xxh_u32)len;
2787
+
2788
+ return XXH32_finalize(h32, input, len&15, align);
2789
+}
2790
+
2791
+/*! @ingroup XXH32_family */
2792
+XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
2793
+{
2794
+#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
2795
+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
2796
+ XXH32_state_t state;
2797
+ XXH32_reset(&state, seed);
2798
+ XXH32_update(&state, (const xxh_u8*)input, len);
2799
+ return XXH32_digest(&state);
2800
+#else
2801
+ if (XXH_FORCE_ALIGN_CHECK) {
2802
+ if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
2803
+ return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
2804
+ } }
2805
+
2806
+ return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
2807
+#endif
2808
+}
2809
+
2810
+
2811
+
2812
+/******* Hash streaming *******/
2813
+#ifndef XXH_NO_STREAM
2814
+/*! @ingroup XXH32_family */
2815
+XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
2816
+{
2817
+ return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
2818
+}
2819
+/*! @ingroup XXH32_family */
2820
+XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
2821
+{
2822
+ XXH_free(statePtr);
2823
+ return XXH_OK;
2824
+}
2825
+
2826
+/*! @ingroup XXH32_family */
2827
+XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
2828
+{
2829
+ XXH_memcpy(dstState, srcState, sizeof(*dstState));
2830
+}
2831
+
2832
+/*! @ingroup XXH32_family */
2833
+XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
2834
+{
2835
+ XXH_ASSERT(statePtr != NULL);
2836
+ memset(statePtr, 0, sizeof(*statePtr));
2837
+ statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
2838
+ statePtr->v[1] = seed + XXH_PRIME32_2;
2839
+ statePtr->v[2] = seed + 0;
2840
+ statePtr->v[3] = seed - XXH_PRIME32_1;
2841
+ return XXH_OK;
2842
+}
2843
+
2844
+
2845
+/*! @ingroup XXH32_family */
2846
+XXH_PUBLIC_API XXH_errorcode
2847
+XXH32_update(XXH32_state_t* state, const void* input, size_t len)
2848
+{
2849
+ if (input==NULL) {
2850
+ XXH_ASSERT(len == 0);
2851
+ return XXH_OK;
2852
+ }
2853
+
2854
+ { const xxh_u8* p = (const xxh_u8*)input;
2855
+ const xxh_u8* const bEnd = p + len;
2856
+
2857
+ state->total_len_32 += (XXH32_hash_t)len;
2858
+ state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
2859
+
2860
+ if (state->memsize + len < 16) { /* fill in tmp buffer */
2861
+ XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
2862
+ state->memsize += (XXH32_hash_t)len;
2863
+ return XXH_OK;
2864
+ }
2865
+
2866
+ if (state->memsize) { /* some data left from previous update */
2867
+ XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
2868
+ { const xxh_u32* p32 = state->mem32;
2869
+ state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++;
2870
+ state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++;
2871
+ state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++;
2872
+ state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32));
2873
+ }
2874
+ p += 16-state->memsize;
2875
+ state->memsize = 0;
2876
+ }
2877
+
2878
+ if (p <= bEnd-16) {
2879
+ const xxh_u8* const limit = bEnd - 16;
2880
+
2881
+ do {
2882
+ state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4;
2883
+ state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4;
2884
+ state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4;
2885
+ state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4;
2886
+ } while (p<=limit);
2887
+
2888
+ }
2889
+
2890
+ if (p < bEnd) {
2891
+ XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
2892
+ state->memsize = (unsigned)(bEnd-p);
2893
+ }
2894
+ }
2895
+
2896
+ return XXH_OK;
2897
+}
2898
+
2899
+
2900
+/*! @ingroup XXH32_family */
2901
+XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state)
2902
+{
2903
+ xxh_u32 h32;
2904
+
2905
+ if (state->large_len) {
2906
+ h32 = XXH_rotl32(state->v[0], 1)
2907
+ + XXH_rotl32(state->v[1], 7)
2908
+ + XXH_rotl32(state->v[2], 12)
2909
+ + XXH_rotl32(state->v[3], 18);
2910
+ } else {
2911
+ h32 = state->v[2] /* == seed */ + XXH_PRIME32_5;
2912
+ }
2913
+
2914
+ h32 += state->total_len_32;
2915
+
2916
+ return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
2917
+}
2918
+#endif /* !XXH_NO_STREAM */
2919
+
2920
+/******* Canonical representation *******/
2921
+
2922
+/*!
2923
+ * @ingroup XXH32_family
2924
+ * The default return values from XXH functions are unsigned 32 and 64 bit
2925
+ * integers.
2926
+ *
2927
+ * The canonical representation uses big endian convention, the same convention
2928
+ * as human-readable numbers (large digits first).
2929
+ *
2930
+ * This way, hash values can be written into a file or buffer, remaining
2931
+ * comparable across different systems.
2932
+ *
2933
+ * The following functions allow transformation of hash values to and from their
2934
+ * canonical format.
2935
+ */
2936
+XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
2937
+{
2938
+ XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
2939
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
2940
+ XXH_memcpy(dst, &hash, sizeof(*dst));
2941
+}
2942
+/*! @ingroup XXH32_family */
2943
+XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
2944
+{
2945
+ return XXH_readBE32(src);
2946
+}
2947
+
2948
+
2949
+#ifndef XXH_NO_LONG_LONG
2950
+
2951
+/* *******************************************************************
2952
+* 64-bit hash functions
2953
+*********************************************************************/
2954
+/*!
2955
+ * @}
2956
+ * @ingroup impl
2957
+ * @{
2958
+ */
2959
+/******* Memory access *******/
2960
+
2961
+typedef XXH64_hash_t xxh_u64;
2962
+
2963
+#ifdef XXH_OLD_NAMES
2964
+# define U64 xxh_u64
2965
+#endif
2966
+
2967
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
2968
+/*
2969
+ * Manual byteshift. Best for old compilers which don't inline memcpy.
2970
+ * We actually directly use XXH_readLE64 and XXH_readBE64.
2971
+ */
2972
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
2973
+
2974
+/* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
2975
+static xxh_u64 XXH_read64(const void* memPtr)
2976
+{
2977
+ return *(const xxh_u64*) memPtr;
2978
+}
2979
+
2980
+#elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
2981
+
2982
+/*
2983
+ * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
2984
+ * documentation claimed that it only increased the alignment, but actually it
2985
+ * can decrease it on gcc, clang, and icc:
2986
+ * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
2987
+ * https://gcc.godbolt.org/z/xYez1j67Y.
2988
+ */
2989
+#ifdef XXH_OLD_NAMES
2990
+typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64;
2991
+#endif
2992
+static xxh_u64 XXH_read64(const void* ptr)
2993
+{
2994
+ typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64;
2995
+ return *((const xxh_unalign64*)ptr);
2996
+}
2997
+
2998
+#else
2999
+
3000
+/*
3001
+ * Portable and safe solution. Generally efficient.
3002
+ * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
3003
+ */
3004
+static xxh_u64 XXH_read64(const void* memPtr)
3005
+{
3006
+ xxh_u64 val;
3007
+ XXH_memcpy(&val, memPtr, sizeof(val));
3008
+ return val;
3009
+}
3010
+
3011
+#endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
3012
+
3013
+#if defined(_MSC_VER) /* Visual Studio */
3014
+# define XXH_swap64 _byteswap_uint64
3015
+#elif XXH_GCC_VERSION >= 403
3016
+# define XXH_swap64 __builtin_bswap64
3017
+#else
3018
+static xxh_u64 XXH_swap64(xxh_u64 x)
3019
+{
3020
+ return ((x << 56) & 0xff00000000000000ULL) |
3021
+ ((x << 40) & 0x00ff000000000000ULL) |
3022
+ ((x << 24) & 0x0000ff0000000000ULL) |
3023
+ ((x << 8) & 0x000000ff00000000ULL) |
3024
+ ((x >> 8) & 0x00000000ff000000ULL) |
3025
+ ((x >> 24) & 0x0000000000ff0000ULL) |
3026
+ ((x >> 40) & 0x000000000000ff00ULL) |
3027
+ ((x >> 56) & 0x00000000000000ffULL);
3028
+}
3029
+#endif
3030
+
3031
+
3032
+/* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
3033
+#if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
3034
+
3035
+XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
3036
+{
3037
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
3038
+ return bytePtr[0]
3039
+ | ((xxh_u64)bytePtr[1] << 8)
3040
+ | ((xxh_u64)bytePtr[2] << 16)
3041
+ | ((xxh_u64)bytePtr[3] << 24)
3042
+ | ((xxh_u64)bytePtr[4] << 32)
3043
+ | ((xxh_u64)bytePtr[5] << 40)
3044
+ | ((xxh_u64)bytePtr[6] << 48)
3045
+ | ((xxh_u64)bytePtr[7] << 56);
3046
+}
3047
+
3048
+XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
3049
+{
3050
+ const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
3051
+ return bytePtr[7]
3052
+ | ((xxh_u64)bytePtr[6] << 8)
3053
+ | ((xxh_u64)bytePtr[5] << 16)
3054
+ | ((xxh_u64)bytePtr[4] << 24)
3055
+ | ((xxh_u64)bytePtr[3] << 32)
3056
+ | ((xxh_u64)bytePtr[2] << 40)
3057
+ | ((xxh_u64)bytePtr[1] << 48)
3058
+ | ((xxh_u64)bytePtr[0] << 56);
3059
+}
3060
+
3061
+#else
3062
+XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
3063
+{
3064
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
3065
+}
3066
+
3067
+static xxh_u64 XXH_readBE64(const void* ptr)
3068
+{
3069
+ return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
3070
+}
3071
+#endif
3072
+
3073
+XXH_FORCE_INLINE xxh_u64
3074
+XXH_readLE64_align(const void* ptr, XXH_alignment align)
3075
+{
3076
+ if (align==XXH_unaligned)
3077
+ return XXH_readLE64(ptr);
3078
+ else
3079
+ return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr);
3080
+}
3081
+
3082
+
3083
+/******* xxh64 *******/
3084
+/*!
3085
+ * @}
3086
+ * @defgroup XXH64_impl XXH64 implementation
3087
+ * @ingroup impl
3088
+ *
3089
+ * Details on the XXH64 implementation.
3090
+ * @{
3091
+ */
3092
+/* #define rather that static const, to be used as initializers */
3093
+#define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */
3094
+#define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */
3095
+#define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */
3096
+#define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */
3097
+#define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */
3098
+
3099
+#ifdef XXH_OLD_NAMES
3100
+# define PRIME64_1 XXH_PRIME64_1
3101
+# define PRIME64_2 XXH_PRIME64_2
3102
+# define PRIME64_3 XXH_PRIME64_3
3103
+# define PRIME64_4 XXH_PRIME64_4
3104
+# define PRIME64_5 XXH_PRIME64_5
3105
+#endif
3106
+
3107
+/*! @copydoc XXH32_round */
3108
+static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
3109
+{
3110
+ acc += input * XXH_PRIME64_2;
3111
+ acc = XXH_rotl64(acc, 31);
3112
+ acc *= XXH_PRIME64_1;
3113
+ return acc;
3114
+}
3115
+
3116
+static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
3117
+{
3118
+ val = XXH64_round(0, val);
3119
+ acc ^= val;
3120
+ acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
3121
+ return acc;
3122
+}
3123
+
3124
+/*! @copydoc XXH32_avalanche */
3125
+static xxh_u64 XXH64_avalanche(xxh_u64 hash)
3126
+{
3127
+ hash ^= hash >> 33;
3128
+ hash *= XXH_PRIME64_2;
3129
+ hash ^= hash >> 29;
3130
+ hash *= XXH_PRIME64_3;
3131
+ hash ^= hash >> 32;
3132
+ return hash;
3133
+}
3134
+
3135
+
3136
+#define XXH_get64bits(p) XXH_readLE64_align(p, align)
3137
+
3138
+/*!
3139
+ * @internal
3140
+ * @brief Processes the last 0-31 bytes of @p ptr.
3141
+ *
3142
+ * There may be up to 31 bytes remaining to consume from the input.
3143
+ * This final stage will digest them to ensure that all input bytes are present
3144
+ * in the final mix.
3145
+ *
3146
+ * @param hash The hash to finalize.
3147
+ * @param ptr The pointer to the remaining input.
3148
+ * @param len The remaining length, modulo 32.
3149
+ * @param align Whether @p ptr is aligned.
3150
+ * @return The finalized hash
3151
+ * @see XXH32_finalize().
3152
+ */
3153
+static XXH_PUREF xxh_u64
3154
+XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
3155
+{
3156
+ if (ptr==NULL) XXH_ASSERT(len == 0);
3157
+ len &= 31;
3158
+ while (len >= 8) {
3159
+ xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr));
3160
+ ptr += 8;
3161
+ hash ^= k1;
3162
+ hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4;
3163
+ len -= 8;
3164
+ }
3165
+ if (len >= 4) {
3166
+ hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;
3167
+ ptr += 4;
3168
+ hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
3169
+ len -= 4;
3170
+ }
3171
+ while (len > 0) {
3172
+ hash ^= (*ptr++) * XXH_PRIME64_5;
3173
+ hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1;
3174
+ --len;
3175
+ }
3176
+ return XXH64_avalanche(hash);
3177
+}
3178
+
3179
+#ifdef XXH_OLD_NAMES
3180
+# define PROCESS1_64 XXH_PROCESS1_64
3181
+# define PROCESS4_64 XXH_PROCESS4_64
3182
+# define PROCESS8_64 XXH_PROCESS8_64
3183
+#else
3184
+# undef XXH_PROCESS1_64
3185
+# undef XXH_PROCESS4_64
3186
+# undef XXH_PROCESS8_64
3187
+#endif
3188
+
3189
+/*!
3190
+ * @internal
3191
+ * @brief The implementation for @ref XXH64().
3192
+ *
3193
+ * @param input , len , seed Directly passed from @ref XXH64().
3194
+ * @param align Whether @p input is aligned.
3195
+ * @return The calculated hash.
3196
+ */
3197
+XXH_FORCE_INLINE XXH_PUREF xxh_u64
3198
+XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
3199
+{
3200
+ xxh_u64 h64;
3201
+ if (input==NULL) XXH_ASSERT(len == 0);
3202
+
3203
+ if (len>=32) {
3204
+ const xxh_u8* const bEnd = input + len;
3205
+ const xxh_u8* const limit = bEnd - 31;
3206
+ xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
3207
+ xxh_u64 v2 = seed + XXH_PRIME64_2;
3208
+ xxh_u64 v3 = seed + 0;
3209
+ xxh_u64 v4 = seed - XXH_PRIME64_1;
3210
+
3211
+ do {
3212
+ v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
3213
+ v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
3214
+ v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
3215
+ v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
3216
+ } while (input<limit);
3217
+
3218
+ h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
3219
+ h64 = XXH64_mergeRound(h64, v1);
3220
+ h64 = XXH64_mergeRound(h64, v2);
3221
+ h64 = XXH64_mergeRound(h64, v3);
3222
+ h64 = XXH64_mergeRound(h64, v4);
3223
+
3224
+ } else {
3225
+ h64 = seed + XXH_PRIME64_5;
3226
+ }
3227
+
3228
+ h64 += (xxh_u64) len;
3229
+
3230
+ return XXH64_finalize(h64, input, len, align);
3231
+}
3232
+
3233
+
3234
+/*! @ingroup XXH64_family */
3235
+XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
3236
+{
3237
+#if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
3238
+ /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
3239
+ XXH64_state_t state;
3240
+ XXH64_reset(&state, seed);
3241
+ XXH64_update(&state, (const xxh_u8*)input, len);
3242
+ return XXH64_digest(&state);
3243
+#else
3244
+ if (XXH_FORCE_ALIGN_CHECK) {
3245
+ if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
3246
+ return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
3247
+ } }
3248
+
3249
+ return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
3250
+
3251
+#endif
3252
+}
3253
+
3254
+/******* Hash Streaming *******/
3255
+#ifndef XXH_NO_STREAM
3256
+/*! @ingroup XXH64_family*/
3257
+XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
3258
+{
3259
+ return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
3260
+}
3261
+/*! @ingroup XXH64_family */
3262
+XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
3263
+{
3264
+ XXH_free(statePtr);
3265
+ return XXH_OK;
3266
+}
3267
+
3268
+/*! @ingroup XXH64_family */
3269
+XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState)
3270
+{
3271
+ XXH_memcpy(dstState, srcState, sizeof(*dstState));
3272
+}
3273
+
3274
+/*! @ingroup XXH64_family */
3275
+XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed)
3276
+{
3277
+ XXH_ASSERT(statePtr != NULL);
3278
+ memset(statePtr, 0, sizeof(*statePtr));
3279
+ statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
3280
+ statePtr->v[1] = seed + XXH_PRIME64_2;
3281
+ statePtr->v[2] = seed + 0;
3282
+ statePtr->v[3] = seed - XXH_PRIME64_1;
3283
+ return XXH_OK;
3284
+}
3285
+
3286
+/*! @ingroup XXH64_family */
3287
+XXH_PUBLIC_API XXH_errorcode
3288
+XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len)
3289
+{
3290
+ if (input==NULL) {
3291
+ XXH_ASSERT(len == 0);
3292
+ return XXH_OK;
3293
+ }
3294
+
3295
+ { const xxh_u8* p = (const xxh_u8*)input;
3296
+ const xxh_u8* const bEnd = p + len;
3297
+
3298
+ state->total_len += len;
3299
+
3300
+ if (state->memsize + len < 32) { /* fill in tmp buffer */
3301
+ XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
3302
+ state->memsize += (xxh_u32)len;
3303
+ return XXH_OK;
3304
+ }
3305
+
3306
+ if (state->memsize) { /* tmp buffer is full */
3307
+ XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
3308
+ state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0));
3309
+ state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1));
3310
+ state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2));
3311
+ state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3));
3312
+ p += 32 - state->memsize;
3313
+ state->memsize = 0;
3314
+ }
3315
+
3316
+ if (p+32 <= bEnd) {
3317
+ const xxh_u8* const limit = bEnd - 32;
3318
+
3319
+ do {
3320
+ state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8;
3321
+ state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8;
3322
+ state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8;
3323
+ state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8;
3324
+ } while (p<=limit);
3325
+
3326
+ }
3327
+
3328
+ if (p < bEnd) {
3329
+ XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
3330
+ state->memsize = (unsigned)(bEnd-p);
3331
+ }
3332
+ }
3333
+
3334
+ return XXH_OK;
3335
+}
3336
+
3337
+
3338
+/*! @ingroup XXH64_family */
3339
+XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state)
3340
+{
3341
+ xxh_u64 h64;
3342
+
3343
+ if (state->total_len >= 32) {
3344
+ h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18);
3345
+ h64 = XXH64_mergeRound(h64, state->v[0]);
3346
+ h64 = XXH64_mergeRound(h64, state->v[1]);
3347
+ h64 = XXH64_mergeRound(h64, state->v[2]);
3348
+ h64 = XXH64_mergeRound(h64, state->v[3]);
3349
+ } else {
3350
+ h64 = state->v[2] /*seed*/ + XXH_PRIME64_5;
3351
+ }
3352
+
3353
+ h64 += (xxh_u64) state->total_len;
3354
+
3355
+ return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
3356
+}
3357
+#endif /* !XXH_NO_STREAM */
3358
+
3359
+/******* Canonical representation *******/
3360
+
3361
+/*! @ingroup XXH64_family */
3362
+XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash)
3363
+{
3364
+ XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
3365
+ if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
3366
+ XXH_memcpy(dst, &hash, sizeof(*dst));
3367
+}
3368
+
3369
+/*! @ingroup XXH64_family */
3370
+XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src)
3371
+{
3372
+ return XXH_readBE64(src);
3373
+}
3374
+
3375
+#ifndef XXH_NO_XXH3
3376
+
3377
+/* *********************************************************************
3378
+* XXH3
3379
+* New generation hash designed for speed on small keys and vectorization
3380
+************************************************************************ */
3381
+/*!
3382
+ * @}
3383
+ * @defgroup XXH3_impl XXH3 implementation
3384
+ * @ingroup impl
3385
+ * @{
3386
+ */
3387
+
3388
+/* === Compiler specifics === */
3389
+
3390
+#if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
3391
+# define XXH_RESTRICT /* disable */
3392
+#elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
3393
+# define XXH_RESTRICT restrict
3394
+#elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \
3395
+ || (defined (__clang__)) \
3396
+ || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \
3397
+ || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300))
3398
+/*
3399
+ * There are a LOT more compilers that recognize __restrict but this
3400
+ * covers the major ones.
3401
+ */
3402
+# define XXH_RESTRICT __restrict
3403
+#else
3404
+# define XXH_RESTRICT /* disable */
3405
+#endif
3406
+
3407
+#if (defined(__GNUC__) && (__GNUC__ >= 3)) \
3408
+ || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
3409
+ || defined(__clang__)
3410
+# define XXH_likely(x) __builtin_expect(x, 1)
3411
+# define XXH_unlikely(x) __builtin_expect(x, 0)
3412
+#else
3413
+# define XXH_likely(x) (x)
3414
+# define XXH_unlikely(x) (x)
3415
+#endif
3416
+
3417
+#ifndef XXH_HAS_INCLUDE
3418
+# ifdef __has_include
3419
+# define XXH_HAS_INCLUDE(x) __has_include(x)
3420
+# else
3421
+# define XXH_HAS_INCLUDE(x) 0
3422
+# endif
3423
+#endif
3424
+
3425
+#if defined(__GNUC__) || defined(__clang__)
3426
+# if defined(__ARM_FEATURE_SVE)
3427
+# include <arm_sve.h>
3428
+# endif
3429
+# if defined(__ARM_NEON__) || defined(__ARM_NEON) \
3430
+ || (defined(_M_ARM) && _M_ARM >= 7) \
3431
+ || defined(_M_ARM64) || defined(_M_ARM64EC) \
3432
+ || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* WASM SIMD128 via SIMDe */
3433
+# define inline __inline__ /* circumvent a clang bug */
3434
+# include <arm_neon.h>
3435
+# undef inline
3436
+# elif defined(__AVX2__)
3437
+# include <immintrin.h>
3438
+# elif defined(__SSE2__)
3439
+# include <emmintrin.h>
3440
+# endif
3441
+#endif
3442
+
3443
+#if defined(_MSC_VER)
3444
+# include <intrin.h>
3445
+#endif
3446
+
3447
+/*
3448
+ * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
3449
+ * remaining a true 64-bit/128-bit hash function.
3450
+ *
3451
+ * This is done by prioritizing a subset of 64-bit operations that can be
3452
+ * emulated without too many steps on the average 32-bit machine.
3453
+ *
3454
+ * For example, these two lines seem similar, and run equally fast on 64-bit:
3455
+ *
3456
+ * xxh_u64 x;
3457
+ * x ^= (x >> 47); // good
3458
+ * x ^= (x >> 13); // bad
3459
+ *
3460
+ * However, to a 32-bit machine, there is a major difference.
3461
+ *
3462
+ * x ^= (x >> 47) looks like this:
3463
+ *
3464
+ * x.lo ^= (x.hi >> (47 - 32));
3465
+ *
3466
+ * while x ^= (x >> 13) looks like this:
3467
+ *
3468
+ * // note: funnel shifts are not usually cheap.
3469
+ * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13));
3470
+ * x.hi ^= (x.hi >> 13);
3471
+ *
3472
+ * The first one is significantly faster than the second, simply because the
3473
+ * shift is larger than 32. This means:
3474
+ * - All the bits we need are in the upper 32 bits, so we can ignore the lower
3475
+ * 32 bits in the shift.
3476
+ * - The shift result will always fit in the lower 32 bits, and therefore,
3477
+ * we can ignore the upper 32 bits in the xor.
3478
+ *
3479
+ * Thanks to this optimization, XXH3 only requires these features to be efficient:
3480
+ *
3481
+ * - Usable unaligned access
3482
+ * - A 32-bit or 64-bit ALU
3483
+ * - If 32-bit, a decent ADC instruction
3484
+ * - A 32 or 64-bit multiply with a 64-bit result
3485
+ * - For the 128-bit variant, a decent byteswap helps short inputs.
3486
+ *
3487
+ * The first two are already required by XXH32, and almost all 32-bit and 64-bit
3488
+ * platforms which can run XXH32 can run XXH3 efficiently.
3489
+ *
3490
+ * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
3491
+ * notable exception.
3492
+ *
3493
+ * First of all, Thumb-1 lacks support for the UMULL instruction which
3494
+ * performs the important long multiply. This means numerous __aeabi_lmul
3495
+ * calls.
3496
+ *
3497
+ * Second of all, the 8 functional registers are just not enough.
3498
+ * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
3499
+ * Lo registers, and this shuffling results in thousands more MOVs than A32.
3500
+ *
3501
+ * A32 and T32 don't have this limitation. They can access all 14 registers,
3502
+ * do a 32->64 multiply with UMULL, and the flexible operand allowing free
3503
+ * shifts is helpful, too.
3504
+ *
3505
+ * Therefore, we do a quick sanity check.
3506
+ *
3507
+ * If compiling Thumb-1 for a target which supports ARM instructions, we will
3508
+ * emit a warning, as it is not a "sane" platform to compile for.
3509
+ *
3510
+ * Usually, if this happens, it is because of an accident and you probably need
3511
+ * to specify -march, as you likely meant to compile for a newer architecture.
3512
+ *
3513
+ * Credit: large sections of the vectorial and asm source code paths
3514
+ * have been contributed by @easyaspi314
3515
+ */
3516
+#if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
3517
+# warning "XXH3 is highly inefficient without ARM or Thumb-2."
3518
+#endif
3519
+
3520
+/* ==========================================
3521
+ * Vectorization detection
3522
+ * ========================================== */
3523
+
3524
+#ifdef XXH_DOXYGEN
3525
+/*!
3526
+ * @ingroup tuning
3527
+ * @brief Overrides the vectorization implementation chosen for XXH3.
3528
+ *
3529
+ * Can be defined to 0 to disable SIMD or any of the values mentioned in
3530
+ * @ref XXH_VECTOR_TYPE.
3531
+ *
3532
+ * If this is not defined, it uses predefined macros to determine the best
3533
+ * implementation.
3534
+ */
3535
+# define XXH_VECTOR XXH_SCALAR
3536
+/*!
3537
+ * @ingroup tuning
3538
+ * @brief Possible values for @ref XXH_VECTOR.
3539
+ *
3540
+ * Note that these are actually implemented as macros.
3541
+ *
3542
+ * If this is not defined, it is detected automatically.
3543
+ * internal macro XXH_X86DISPATCH overrides this.
3544
+ */
3545
+enum XXH_VECTOR_TYPE /* fake enum */ {
3546
+ XXH_SCALAR = 0, /*!< Portable scalar version */
3547
+ XXH_SSE2 = 1, /*!<
3548
+ * SSE2 for Pentium 4, Opteron, all x86_64.
3549
+ *
3550
+ * @note SSE2 is also guaranteed on Windows 10, macOS, and
3551
+ * Android x86.
3552
+ */
3553
+ XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */
3554
+ XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */
3555
+ XXH_NEON = 4, /*!<
3556
+ * NEON for most ARMv7-A, all AArch64, and WASM SIMD128
3557
+ * via the SIMDeverywhere polyfill provided with the
3558
+ * Emscripten SDK.
3559
+ */
3560
+ XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */
3561
+ XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */
3562
+};
3563
+/*!
3564
+ * @ingroup tuning
3565
+ * @brief Selects the minimum alignment for XXH3's accumulators.
3566
+ *
3567
+ * When using SIMD, this should match the alignment required for said vector
3568
+ * type, so, for example, 32 for AVX2.
3569
+ *
3570
+ * Default: Auto detected.
3571
+ */
3572
+# define XXH_ACC_ALIGN 8
3573
+#endif
3574
+
3575
+/* Actual definition */
3576
+#ifndef XXH_DOXYGEN
3577
+# define XXH_SCALAR 0
3578
+# define XXH_SSE2 1
3579
+# define XXH_AVX2 2
3580
+# define XXH_AVX512 3
3581
+# define XXH_NEON 4
3582
+# define XXH_VSX 5
3583
+# define XXH_SVE 6
3584
+#endif
3585
+
3586
+#ifndef XXH_VECTOR /* can be defined on command line */
3587
+# if defined(__ARM_FEATURE_SVE)
3588
+# define XXH_VECTOR XXH_SVE
3589
+# elif ( \
3590
+ defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \
3591
+ || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \
3592
+ || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* wasm simd128 via SIMDe */ \
3593
+ ) && ( \
3594
+ defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \
3595
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
3596
+ )
3597
+# define XXH_VECTOR XXH_NEON
3598
+# elif defined(__AVX512F__)
3599
+# define XXH_VECTOR XXH_AVX512
3600
+# elif defined(__AVX2__)
3601
+# define XXH_VECTOR XXH_AVX2
3602
+# elif defined(__SSE2__) || defined(_M_AMD64) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
3603
+# define XXH_VECTOR XXH_SSE2
3604
+# elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
3605
+ || (defined(__s390x__) && defined(__VEC__)) \
3606
+ && defined(__GNUC__) /* TODO: IBM XL */
3607
+# define XXH_VECTOR XXH_VSX
3608
+# else
3609
+# define XXH_VECTOR XXH_SCALAR
3610
+# endif
3611
+#endif
3612
+
3613
+/* __ARM_FEATURE_SVE is only supported by GCC & Clang. */
3614
+#if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE)
3615
+# ifdef _MSC_VER
3616
+# pragma warning(once : 4606)
3617
+# else
3618
+# warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead."
3619
+# endif
3620
+# undef XXH_VECTOR
3621
+# define XXH_VECTOR XXH_SCALAR
3622
+#endif
3623
+
3624
+/*
3625
+ * Controls the alignment of the accumulator,
3626
+ * for compatibility with aligned vector loads, which are usually faster.
3627
+ */
3628
+#ifndef XXH_ACC_ALIGN
3629
+# if defined(XXH_X86DISPATCH)
3630
+# define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */
3631
+# elif XXH_VECTOR == XXH_SCALAR /* scalar */
3632
+# define XXH_ACC_ALIGN 8
3633
+# elif XXH_VECTOR == XXH_SSE2 /* sse2 */
3634
+# define XXH_ACC_ALIGN 16
3635
+# elif XXH_VECTOR == XXH_AVX2 /* avx2 */
3636
+# define XXH_ACC_ALIGN 32
3637
+# elif XXH_VECTOR == XXH_NEON /* neon */
3638
+# define XXH_ACC_ALIGN 16
3639
+# elif XXH_VECTOR == XXH_VSX /* vsx */
3640
+# define XXH_ACC_ALIGN 16
3641
+# elif XXH_VECTOR == XXH_AVX512 /* avx512 */
3642
+# define XXH_ACC_ALIGN 64
3643
+# elif XXH_VECTOR == XXH_SVE /* sve */
3644
+# define XXH_ACC_ALIGN 64
3645
+# endif
3646
+#endif
3647
+
3648
+#if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \
3649
+ || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512
3650
+# define XXH_SEC_ALIGN XXH_ACC_ALIGN
3651
+#elif XXH_VECTOR == XXH_SVE
3652
+# define XXH_SEC_ALIGN XXH_ACC_ALIGN
3653
+#else
3654
+# define XXH_SEC_ALIGN 8
3655
+#endif
3656
+
3657
+#if defined(__GNUC__) || defined(__clang__)
3658
+# define XXH_ALIASING __attribute__((may_alias))
3659
+#else
3660
+# define XXH_ALIASING /* nothing */
3661
+#endif
3662
+
3663
+/*
3664
+ * UGLY HACK:
3665
+ * GCC usually generates the best code with -O3 for xxHash.
3666
+ *
3667
+ * However, when targeting AVX2, it is overzealous in its unrolling resulting
3668
+ * in code roughly 3/4 the speed of Clang.
3669
+ *
3670
+ * There are other issues, such as GCC splitting _mm256_loadu_si256 into
3671
+ * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
3672
+ * only applies to Sandy and Ivy Bridge... which don't even support AVX2.
3673
+ *
3674
+ * That is why when compiling the AVX2 version, it is recommended to use either
3675
+ * -O2 -mavx2 -march=haswell
3676
+ * or
3677
+ * -O2 -mavx2 -mno-avx256-split-unaligned-load
3678
+ * for decent performance, or to use Clang instead.
3679
+ *
3680
+ * Fortunately, we can control the first one with a pragma that forces GCC into
3681
+ * -O2, but the other one we can't control without "failed to inline always
3682
+ * inline function due to target mismatch" warnings.
3683
+ */
3684
+#if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
3685
+ && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
3686
+ && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
3687
+# pragma GCC push_options
3688
+# pragma GCC optimize("-O2")
3689
+#endif
3690
+
3691
+#if XXH_VECTOR == XXH_NEON
3692
+
3693
+/*
3694
+ * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3
3695
+ * optimizes out the entire hashLong loop because of the aliasing violation.
3696
+ *
3697
+ * However, GCC is also inefficient at load-store optimization with vld1q/vst1q,
3698
+ * so the only option is to mark it as aliasing.
3699
+ */
3700
+typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING;
3701
+
3702
+/*!
3703
+ * @internal
3704
+ * @brief `vld1q_u64` but faster and alignment-safe.
3705
+ *
3706
+ * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only
3707
+ * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86).
3708
+ *
3709
+ * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it
3710
+ * prohibits load-store optimizations. Therefore, a direct dereference is used.
3711
+ *
3712
+ * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe
3713
+ * unaligned load.
3714
+ */
3715
+#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)
3716
+XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */
3717
+{
3718
+ return *(xxh_aliasing_uint64x2_t const *)ptr;
3719
+}
3720
+#else
3721
+XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr)
3722
+{
3723
+ return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr));
3724
+}
3725
+#endif
3726
+
3727
+/*!
3728
+ * @internal
3729
+ * @brief `vmlal_u32` on low and high halves of a vector.
3730
+ *
3731
+ * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with
3732
+ * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32`
3733
+ * with `vmlal_u32`.
3734
+ */
3735
+#if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11
3736
+XXH_FORCE_INLINE uint64x2_t
3737
+XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
3738
+{
3739
+ /* Inline assembly is the only way */
3740
+ __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs));
3741
+ return acc;
3742
+}
3743
+XXH_FORCE_INLINE uint64x2_t
3744
+XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
3745
+{
3746
+ /* This intrinsic works as expected */
3747
+ return vmlal_high_u32(acc, lhs, rhs);
3748
+}
3749
+#else
3750
+/* Portable intrinsic versions */
3751
+XXH_FORCE_INLINE uint64x2_t
3752
+XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
3753
+{
3754
+ return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs));
3755
+}
3756
+/*! @copydoc XXH_vmlal_low_u32
3757
+ * Assume the compiler converts this to vmlal_high_u32 on aarch64 */
3758
+XXH_FORCE_INLINE uint64x2_t
3759
+XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
3760
+{
3761
+ return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs));
3762
+}
3763
+#endif
3764
+
3765
+/*!
3766
+ * @ingroup tuning
3767
+ * @brief Controls the NEON to scalar ratio for XXH3
3768
+ *
3769
+ * This can be set to 2, 4, 6, or 8.
3770
+ *
3771
+ * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used.
3772
+ *
3773
+ * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those
3774
+ * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU
3775
+ * bandwidth.
3776
+ *
3777
+ * This is even more noticeable on the more advanced cores like the Cortex-A76 which
3778
+ * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once.
3779
+ *
3780
+ * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes
3781
+ * and 2 scalar lanes, which is chosen by default.
3782
+ *
3783
+ * This does not apply to Apple processors or 32-bit processors, which run better with
3784
+ * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes.
3785
+ *
3786
+ * This change benefits CPUs with large micro-op buffers without negatively affecting
3787
+ * most other CPUs:
3788
+ *
3789
+ * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. |
3790
+ * |:----------------------|:--------------------|----------:|-----------:|------:|
3791
+ * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% |
3792
+ * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% |
3793
+ * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% |
3794
+ * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% |
3795
+ *
3796
+ * It also seems to fix some bad codegen on GCC, making it almost as fast as clang.
3797
+ *
3798
+ * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning
3799
+ * it effectively becomes worse 4.
3800
+ *
3801
+ * @see XXH3_accumulate_512_neon()
3802
+ */
3803
+# ifndef XXH3_NEON_LANES
3804
+# if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \
3805
+ && !defined(__APPLE__) && XXH_SIZE_OPT <= 0
3806
+# define XXH3_NEON_LANES 6
3807
+# else
3808
+# define XXH3_NEON_LANES XXH_ACC_NB
3809
+# endif
3810
+# endif
3811
+#endif /* XXH_VECTOR == XXH_NEON */
3812
+
3813
+/*
3814
+ * VSX and Z Vector helpers.
3815
+ *
3816
+ * This is very messy, and any pull requests to clean this up are welcome.
3817
+ *
3818
+ * There are a lot of problems with supporting VSX and s390x, due to
3819
+ * inconsistent intrinsics, spotty coverage, and multiple endiannesses.
3820
+ */
3821
+#if XXH_VECTOR == XXH_VSX
3822
+/* Annoyingly, these headers _may_ define three macros: `bool`, `vector`,
3823
+ * and `pixel`. This is a problem for obvious reasons.
3824
+ *
3825
+ * These keywords are unnecessary; the spec literally says they are
3826
+ * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd
3827
+ * after including the header.
3828
+ *
3829
+ * We use pragma push_macro/pop_macro to keep the namespace clean. */
3830
+# pragma push_macro("bool")
3831
+# pragma push_macro("vector")
3832
+# pragma push_macro("pixel")
3833
+/* silence potential macro redefined warnings */
3834
+# undef bool
3835
+# undef vector
3836
+# undef pixel
3837
+
3838
+# if defined(__s390x__)
3839
+# include <s390intrin.h>
3840
+# else
3841
+# include <altivec.h>
3842
+# endif
3843
+
3844
+/* Restore the original macro values, if applicable. */
3845
+# pragma pop_macro("pixel")
3846
+# pragma pop_macro("vector")
3847
+# pragma pop_macro("bool")
3848
+
3849
+typedef __vector unsigned long long xxh_u64x2;
3850
+typedef __vector unsigned char xxh_u8x16;
3851
+typedef __vector unsigned xxh_u32x4;
3852
+
3853
+/*
3854
+ * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue.
3855
+ */
3856
+typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING;
3857
+
3858
+# ifndef XXH_VSX_BE
3859
+# if defined(__BIG_ENDIAN__) \
3860
+ || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
3861
+# define XXH_VSX_BE 1
3862
+# elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
3863
+# warning "-maltivec=be is not recommended. Please use native endianness."
3864
+# define XXH_VSX_BE 1
3865
+# else
3866
+# define XXH_VSX_BE 0
3867
+# endif
3868
+# endif /* !defined(XXH_VSX_BE) */
3869
+
3870
+# if XXH_VSX_BE
3871
+# if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
3872
+# define XXH_vec_revb vec_revb
3873
+# else
3874
+/*!
3875
+ * A polyfill for POWER9's vec_revb().
3876
+ */
3877
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
3878
+{
3879
+ xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
3880
+ 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
3881
+ return vec_perm(val, val, vByteSwap);
3882
+}
3883
+# endif
3884
+# endif /* XXH_VSX_BE */
3885
+
3886
+/*!
3887
+ * Performs an unaligned vector load and byte swaps it on big endian.
3888
+ */
3889
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
3890
+{
3891
+ xxh_u64x2 ret;
3892
+ XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2));
3893
+# if XXH_VSX_BE
3894
+ ret = XXH_vec_revb(ret);
3895
+# endif
3896
+ return ret;
3897
+}
3898
+
3899
+/*
3900
+ * vec_mulo and vec_mule are very problematic intrinsics on PowerPC
3901
+ *
3902
+ * These intrinsics weren't added until GCC 8, despite existing for a while,
3903
+ * and they are endian dependent. Also, their meaning swap depending on version.
3904
+ * */
3905
+# if defined(__s390x__)
3906
+ /* s390x is always big endian, no issue on this platform */
3907
+# define XXH_vec_mulo vec_mulo
3908
+# define XXH_vec_mule vec_mule
3909
+# elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__)
3910
+/* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
3911
+ /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */
3912
+# define XXH_vec_mulo __builtin_altivec_vmulouw
3913
+# define XXH_vec_mule __builtin_altivec_vmuleuw
3914
+# else
3915
+/* gcc needs inline assembly */
3916
+/* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
3917
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
3918
+{
3919
+ xxh_u64x2 result;
3920
+ __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
3921
+ return result;
3922
+}
3923
+XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
3924
+{
3925
+ xxh_u64x2 result;
3926
+ __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
3927
+ return result;
3928
+}
3929
+# endif /* XXH_vec_mulo, XXH_vec_mule */
3930
+#endif /* XXH_VECTOR == XXH_VSX */
3931
+
3932
+#if XXH_VECTOR == XXH_SVE
3933
+#define ACCRND(acc, offset) \
3934
+do { \
3935
+ svuint64_t input_vec = svld1_u64(mask, xinput + offset); \
3936
+ svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \
3937
+ svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \
3938
+ svuint64_t swapped = svtbl_u64(input_vec, kSwap); \
3939
+ svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \
3940
+ svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \
3941
+ svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \
3942
+ acc = svadd_u64_x(mask, acc, mul); \
3943
+} while (0)
3944
+#endif /* XXH_VECTOR == XXH_SVE */
3945
+
3946
+/* prefetch
3947
+ * can be disabled, by declaring XXH_NO_PREFETCH build macro */
3948
+#if defined(XXH_NO_PREFETCH)
3949
+# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
3950
+#else
3951
+# if XXH_SIZE_OPT >= 1
3952
+# define XXH_PREFETCH(ptr) (void)(ptr)
3953
+# elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */
3954
+# include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
3955
+# define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
3956
+# elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
3957
+# define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
3958
+# else
3959
+# define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
3960
+# endif
3961
+#endif /* XXH_NO_PREFETCH */
3962
+
3963
+
3964
+/* ==========================================
3965
+ * XXH3 default settings
3966
+ * ========================================== */
3967
+
3968
+#define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */
3969
+
3970
+#if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
3971
+# error "default keyset is not large enough"
3972
+#endif
3973
+
3974
+/*! Pseudorandom secret taken directly from FARSH. */
3975
+XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
3976
+ 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
3977
+ 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
3978
+ 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
3979
+ 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
3980
+ 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
3981
+ 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
3982
+ 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
3983
+ 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
3984
+ 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
3985
+ 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
3986
+ 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
3987
+ 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
3988
+};
3989
+
3990
+static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */
3991
+static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */
3992
+
3993
+#ifdef XXH_OLD_NAMES
3994
+# define kSecret XXH3_kSecret
3995
+#endif
3996
+
3997
+#ifdef XXH_DOXYGEN
3998
+/*!
3999
+ * @brief Calculates a 32-bit to 64-bit long multiply.
4000
+ *
4001
+ * Implemented as a macro.
4002
+ *
4003
+ * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't
4004
+ * need to (but it shouldn't need to anyways, it is about 7 instructions to do
4005
+ * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we
4006
+ * use that instead of the normal method.
4007
+ *
4008
+ * If you are compiling for platforms like Thumb-1 and don't have a better option,
4009
+ * you may also want to write your own long multiply routine here.
4010
+ *
4011
+ * @param x, y Numbers to be multiplied
4012
+ * @return 64-bit product of the low 32 bits of @p x and @p y.
4013
+ */
4014
+XXH_FORCE_INLINE xxh_u64
4015
+XXH_mult32to64(xxh_u64 x, xxh_u64 y)
4016
+{
4017
+ return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
4018
+}
4019
+#elif defined(_MSC_VER) && defined(_M_IX86)
4020
+# define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
4021
+#else
4022
+/*
4023
+ * Downcast + upcast is usually better than masking on older compilers like
4024
+ * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
4025
+ *
4026
+ * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
4027
+ * and perform a full 64x64 multiply -- entirely redundant on 32-bit.
4028
+ */
4029
+# define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
4030
+#endif
4031
+
4032
+/*!
4033
+ * @brief Calculates a 64->128-bit long multiply.
4034
+ *
4035
+ * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar
4036
+ * version.
4037
+ *
4038
+ * @param lhs , rhs The 64-bit integers to be multiplied
4039
+ * @return The 128-bit result represented in an @ref XXH128_hash_t.
4040
+ */
4041
+static XXH128_hash_t
4042
+XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
4043
+{
4044
+ /*
4045
+ * GCC/Clang __uint128_t method.
4046
+ *
4047
+ * On most 64-bit targets, GCC and Clang define a __uint128_t type.
4048
+ * This is usually the best way as it usually uses a native long 64-bit
4049
+ * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
4050
+ *
4051
+ * Usually.
4052
+ *
4053
+ * Despite being a 32-bit platform, Clang (and emscripten) define this type
4054
+ * despite not having the arithmetic for it. This results in a laggy
4055
+ * compiler builtin call which calculates a full 128-bit multiply.
4056
+ * In that case it is best to use the portable one.
4057
+ * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
4058
+ */
4059
+#if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \
4060
+ && defined(__SIZEOF_INT128__) \
4061
+ || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
4062
+
4063
+ __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
4064
+ XXH128_hash_t r128;
4065
+ r128.low64 = (xxh_u64)(product);
4066
+ r128.high64 = (xxh_u64)(product >> 64);
4067
+ return r128;
4068
+
4069
+ /*
4070
+ * MSVC for x64's _umul128 method.
4071
+ *
4072
+ * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
4073
+ *
4074
+ * This compiles to single operand MUL on x64.
4075
+ */
4076
+#elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC)
4077
+
4078
+#ifndef _MSC_VER
4079
+# pragma intrinsic(_umul128)
4080
+#endif
4081
+ xxh_u64 product_high;
4082
+ xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
4083
+ XXH128_hash_t r128;
4084
+ r128.low64 = product_low;
4085
+ r128.high64 = product_high;
4086
+ return r128;
4087
+
4088
+ /*
4089
+ * MSVC for ARM64's __umulh method.
4090
+ *
4091
+ * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method.
4092
+ */
4093
+#elif defined(_M_ARM64) || defined(_M_ARM64EC)
4094
+
4095
+#ifndef _MSC_VER
4096
+# pragma intrinsic(__umulh)
4097
+#endif
4098
+ XXH128_hash_t r128;
4099
+ r128.low64 = lhs * rhs;
4100
+ r128.high64 = __umulh(lhs, rhs);
4101
+ return r128;
4102
+
4103
+#else
4104
+ /*
4105
+ * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
4106
+ *
4107
+ * This is a fast and simple grade school multiply, which is shown below
4108
+ * with base 10 arithmetic instead of base 0x100000000.
4109
+ *
4110
+ * 9 3 // D2 lhs = 93
4111
+ * x 7 5 // D2 rhs = 75
4112
+ * ----------
4113
+ * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
4114
+ * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45
4115
+ * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21
4116
+ * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63
4117
+ * ---------
4118
+ * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
4119
+ * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
4120
+ * ---------
4121
+ * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
4122
+ *
4123
+ * The reasons for adding the products like this are:
4124
+ * 1. It avoids manual carry tracking. Just like how
4125
+ * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
4126
+ * This avoids a lot of complexity.
4127
+ *
4128
+ * 2. It hints for, and on Clang, compiles to, the powerful UMAAL
4129
+ * instruction available in ARM's Digital Signal Processing extension
4130
+ * in 32-bit ARMv6 and later, which is shown below:
4131
+ *
4132
+ * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
4133
+ * {
4134
+ * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
4135
+ * *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
4136
+ * *RdHi = (xxh_u32)(product >> 32);
4137
+ * }
4138
+ *
4139
+ * This instruction was designed for efficient long multiplication, and
4140
+ * allows this to be calculated in only 4 instructions at speeds
4141
+ * comparable to some 64-bit ALUs.
4142
+ *
4143
+ * 3. It isn't terrible on other platforms. Usually this will be a couple
4144
+ * of 32-bit ADD/ADCs.
4145
+ */
4146
+
4147
+ /* First calculate all of the cross products. */
4148
+ xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
4149
+ xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF);
4150
+ xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
4151
+ xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32);
4152
+
4153
+ /* Now add the products together. These will never overflow. */
4154
+ xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
4155
+ xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi;
4156
+ xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
4157
+
4158
+ XXH128_hash_t r128;
4159
+ r128.low64 = lower;
4160
+ r128.high64 = upper;
4161
+ return r128;
4162
+#endif
4163
+}
4164
+
4165
+/*!
4166
+ * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it.
4167
+ *
4168
+ * The reason for the separate function is to prevent passing too many structs
4169
+ * around by value. This will hopefully inline the multiply, but we don't force it.
4170
+ *
4171
+ * @param lhs , rhs The 64-bit integers to multiply
4172
+ * @return The low 64 bits of the product XOR'd by the high 64 bits.
4173
+ * @see XXH_mult64to128()
4174
+ */
4175
+static xxh_u64
4176
+XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
4177
+{
4178
+ XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
4179
+ return product.low64 ^ product.high64;
4180
+}
4181
+
4182
+/*! Seems to produce slightly better code on GCC for some reason. */
4183
+XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
4184
+{
4185
+ XXH_ASSERT(0 <= shift && shift < 64);
4186
+ return v64 ^ (v64 >> shift);
4187
+}
4188
+
4189
+/*
4190
+ * This is a fast avalanche stage,
4191
+ * suitable when input bits are already partially mixed
4192
+ */
4193
+static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
4194
+{
4195
+ h64 = XXH_xorshift64(h64, 37);
4196
+ h64 *= PRIME_MX1;
4197
+ h64 = XXH_xorshift64(h64, 32);
4198
+ return h64;
4199
+}
4200
+
4201
+/*
4202
+ * This is a stronger avalanche,
4203
+ * inspired by Pelle Evensen's rrmxmx
4204
+ * preferable when input has not been previously mixed
4205
+ */
4206
+static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
4207
+{
4208
+ /* this mix is inspired by Pelle Evensen's rrmxmx */
4209
+ h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
4210
+ h64 *= PRIME_MX2;
4211
+ h64 ^= (h64 >> 35) + len ;
4212
+ h64 *= PRIME_MX2;
4213
+ return XXH_xorshift64(h64, 28);
4214
+}
4215
+
4216
+
4217
+/* ==========================================
4218
+ * Short keys
4219
+ * ==========================================
4220
+ * One of the shortcomings of XXH32 and XXH64 was that their performance was
4221
+ * sub-optimal on short lengths. It used an iterative algorithm which strongly
4222
+ * favored lengths that were a multiple of 4 or 8.
4223
+ *
4224
+ * Instead of iterating over individual inputs, we use a set of single shot
4225
+ * functions which piece together a range of lengths and operate in constant time.
4226
+ *
4227
+ * Additionally, the number of multiplies has been significantly reduced. This
4228
+ * reduces latency, especially when emulating 64-bit multiplies on 32-bit.
4229
+ *
4230
+ * Depending on the platform, this may or may not be faster than XXH32, but it
4231
+ * is almost guaranteed to be faster than XXH64.
4232
+ */
4233
+
4234
+/*
4235
+ * At very short lengths, there isn't enough input to fully hide secrets, or use
4236
+ * the entire secret.
4237
+ *
4238
+ * There is also only a limited amount of mixing we can do before significantly
4239
+ * impacting performance.
4240
+ *
4241
+ * Therefore, we use different sections of the secret and always mix two secret
4242
+ * samples with an XOR. This should have no effect on performance on the
4243
+ * seedless or withSeed variants because everything _should_ be constant folded
4244
+ * by modern compilers.
4245
+ *
4246
+ * The XOR mixing hides individual parts of the secret and increases entropy.
4247
+ *
4248
+ * This adds an extra layer of strength for custom secrets.
4249
+ */
4250
+XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4251
+XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4252
+{
4253
+ XXH_ASSERT(input != NULL);
4254
+ XXH_ASSERT(1 <= len && len <= 3);
4255
+ XXH_ASSERT(secret != NULL);
4256
+ /*
4257
+ * len = 1: combined = { input[0], 0x01, input[0], input[0] }
4258
+ * len = 2: combined = { input[1], 0x02, input[0], input[1] }
4259
+ * len = 3: combined = { input[2], 0x03, input[0], input[1] }
4260
+ */
4261
+ { xxh_u8 const c1 = input[0];
4262
+ xxh_u8 const c2 = input[len >> 1];
4263
+ xxh_u8 const c3 = input[len - 1];
4264
+ xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24)
4265
+ | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
4266
+ xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
4267
+ xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
4268
+ return XXH64_avalanche(keyed);
4269
+ }
4270
+}
4271
+
4272
+XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4273
+XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4274
+{
4275
+ XXH_ASSERT(input != NULL);
4276
+ XXH_ASSERT(secret != NULL);
4277
+ XXH_ASSERT(4 <= len && len <= 8);
4278
+ seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
4279
+ { xxh_u32 const input1 = XXH_readLE32(input);
4280
+ xxh_u32 const input2 = XXH_readLE32(input + len - 4);
4281
+ xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
4282
+ xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
4283
+ xxh_u64 const keyed = input64 ^ bitflip;
4284
+ return XXH3_rrmxmx(keyed, len);
4285
+ }
4286
+}
4287
+
4288
+XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4289
+XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4290
+{
4291
+ XXH_ASSERT(input != NULL);
4292
+ XXH_ASSERT(secret != NULL);
4293
+ XXH_ASSERT(9 <= len && len <= 16);
4294
+ { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
4295
+ xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
4296
+ xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1;
4297
+ xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
4298
+ xxh_u64 const acc = len
4299
+ + XXH_swap64(input_lo) + input_hi
4300
+ + XXH3_mul128_fold64(input_lo, input_hi);
4301
+ return XXH3_avalanche(acc);
4302
+ }
4303
+}
4304
+
4305
+XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4306
+XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4307
+{
4308
+ XXH_ASSERT(len <= 16);
4309
+ { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed);
4310
+ if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
4311
+ if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
4312
+ return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
4313
+ }
4314
+}
4315
+
4316
+/*
4317
+ * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
4318
+ * multiplication by zero, affecting hashes of lengths 17 to 240.
4319
+ *
4320
+ * However, they are very unlikely.
4321
+ *
4322
+ * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
4323
+ * unseeded non-cryptographic hashes, it does not attempt to defend itself
4324
+ * against specially crafted inputs, only random inputs.
4325
+ *
4326
+ * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
4327
+ * cancelling out the secret is taken an arbitrary number of times (addressed
4328
+ * in XXH3_accumulate_512), this collision is very unlikely with random inputs
4329
+ * and/or proper seeding:
4330
+ *
4331
+ * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
4332
+ * function that is only called up to 16 times per hash with up to 240 bytes of
4333
+ * input.
4334
+ *
4335
+ * This is not too bad for a non-cryptographic hash function, especially with
4336
+ * only 64 bit outputs.
4337
+ *
4338
+ * The 128-bit variant (which trades some speed for strength) is NOT affected
4339
+ * by this, although it is always a good idea to use a proper seed if you care
4340
+ * about strength.
4341
+ */
4342
+XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
4343
+ const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
4344
+{
4345
+#if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
4346
+ && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \
4347
+ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */
4348
+ /*
4349
+ * UGLY HACK:
4350
+ * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
4351
+ * slower code.
4352
+ *
4353
+ * By forcing seed64 into a register, we disrupt the cost model and
4354
+ * cause it to scalarize. See `XXH32_round()`
4355
+ *
4356
+ * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
4357
+ * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
4358
+ * GCC 9.2, despite both emitting scalar code.
4359
+ *
4360
+ * GCC generates much better scalar code than Clang for the rest of XXH3,
4361
+ * which is why finding a more optimal codepath is an interest.
4362
+ */
4363
+ XXH_COMPILER_GUARD(seed64);
4364
+#endif
4365
+ { xxh_u64 const input_lo = XXH_readLE64(input);
4366
+ xxh_u64 const input_hi = XXH_readLE64(input+8);
4367
+ return XXH3_mul128_fold64(
4368
+ input_lo ^ (XXH_readLE64(secret) + seed64),
4369
+ input_hi ^ (XXH_readLE64(secret+8) - seed64)
4370
+ );
4371
+ }
4372
+}
4373
+
4374
+/* For mid range keys, XXH3 uses a Mum-hash variant. */
4375
+XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
4376
+XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
4377
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
4378
+ XXH64_hash_t seed)
4379
+{
4380
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
4381
+ XXH_ASSERT(16 < len && len <= 128);
4382
+
4383
+ { xxh_u64 acc = len * XXH_PRIME64_1;
4384
+#if XXH_SIZE_OPT >= 1
4385
+ /* Smaller and cleaner, but slightly slower. */
4386
+ unsigned int i = (unsigned int)(len - 1) / 32;
4387
+ do {
4388
+ acc += XXH3_mix16B(input+16 * i, secret+32*i, seed);
4389
+ acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed);
4390
+ } while (i-- != 0);
4391
+#else
4392
+ if (len > 32) {
4393
+ if (len > 64) {
4394
+ if (len > 96) {
4395
+ acc += XXH3_mix16B(input+48, secret+96, seed);
4396
+ acc += XXH3_mix16B(input+len-64, secret+112, seed);
4397
+ }
4398
+ acc += XXH3_mix16B(input+32, secret+64, seed);
4399
+ acc += XXH3_mix16B(input+len-48, secret+80, seed);
4400
+ }
4401
+ acc += XXH3_mix16B(input+16, secret+32, seed);
4402
+ acc += XXH3_mix16B(input+len-32, secret+48, seed);
4403
+ }
4404
+ acc += XXH3_mix16B(input+0, secret+0, seed);
4405
+ acc += XXH3_mix16B(input+len-16, secret+16, seed);
4406
+#endif
4407
+ return XXH3_avalanche(acc);
4408
+ }
4409
+}
4410
+
4411
+#define XXH3_MIDSIZE_MAX 240
4412
+
4413
+XXH_NO_INLINE XXH_PUREF XXH64_hash_t
4414
+XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
4415
+ const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
4416
+ XXH64_hash_t seed)
4417
+{
4418
+ XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
4419
+ XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
4420
+
4421
+ #define XXH3_MIDSIZE_STARTOFFSET 3
4422
+ #define XXH3_MIDSIZE_LASTOFFSET 17
4423
+
4424
+ { xxh_u64 acc = len * XXH_PRIME64_1;
4425
+ xxh_u64 acc_end;
4426
+ unsigned int const nbRounds = (unsigned int)len / 16;
4427
+ unsigned int i;
4428
+ XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
4429
+ for (i=0; i<8; i++) {
4430
+ acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed);
4431
+ }
4432
+ /* last bytes */
4433
+ acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
4434
+ XXH_ASSERT(nbRounds >= 8);
4435
+ acc = XXH3_avalanche(acc);
4436
+#if defined(__clang__) /* Clang */ \
4437
+ && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
4438
+ && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
4439
+ /*
4440
+ * UGLY HACK:
4441
+ * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
4442
+ * In everywhere else, it uses scalar code.
4443
+ *
4444
+ * For 64->128-bit multiplies, even if the NEON was 100% optimal, it
4445
+ * would still be slower than UMAAL (see XXH_mult64to128).
4446
+ *
4447
+ * Unfortunately, Clang doesn't handle the long multiplies properly and
4448
+ * converts them to the nonexistent "vmulq_u64" intrinsic, which is then
4449
+ * scalarized into an ugly mess of VMOV.32 instructions.
4450
+ *
4451
+ * This mess is difficult to avoid without turning autovectorization
4452
+ * off completely, but they are usually relatively minor and/or not
4453
+ * worth it to fix.
4454
+ *
4455
+ * This loop is the easiest to fix, as unlike XXH32, this pragma
4456
+ * _actually works_ because it is a loop vectorization instead of an
4457
+ * SLP vectorization.
4458
+ */
4459
+ #pragma clang loop vectorize(disable)
4460
+#endif
4461
+ for (i=8 ; i < nbRounds; i++) {
4462
+ /*
4463
+ * Prevents clang for unrolling the acc loop and interleaving with this one.
4464
+ */
4465
+ XXH_COMPILER_GUARD(acc);
4466
+ acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
4467
+ }
4468
+ return XXH3_avalanche(acc + acc_end);
4469
+ }
4470
+}
4471
+
4472
+
4473
+/* ======= Long Keys ======= */
4474
+
4475
+#define XXH_STRIPE_LEN 64
4476
+#define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */
4477
+#define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))
4478
+
4479
+#ifdef XXH_OLD_NAMES
4480
+# define STRIPE_LEN XXH_STRIPE_LEN
4481
+# define ACC_NB XXH_ACC_NB
4482
+#endif
4483
+
4484
+#ifndef XXH_PREFETCH_DIST
4485
+# ifdef __clang__
4486
+# define XXH_PREFETCH_DIST 320
4487
+# else
4488
+# if (XXH_VECTOR == XXH_AVX512)
4489
+# define XXH_PREFETCH_DIST 512
4490
+# else
4491
+# define XXH_PREFETCH_DIST 384
4492
+# endif
4493
+# endif /* __clang__ */
4494
+#endif /* XXH_PREFETCH_DIST */
4495
+
4496
+/*
4497
+ * These macros are to generate an XXH3_accumulate() function.
4498
+ * The two arguments select the name suffix and target attribute.
4499
+ *
4500
+ * The name of this symbol is XXH3_accumulate_<name>() and it calls
4501
+ * XXH3_accumulate_512_<name>().
4502
+ *
4503
+ * It may be useful to hand implement this function if the compiler fails to
4504
+ * optimize the inline function.
4505
+ */
4506
+#define XXH3_ACCUMULATE_TEMPLATE(name) \
4507
+void \
4508
+XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \
4509
+ const xxh_u8* XXH_RESTRICT input, \
4510
+ const xxh_u8* XXH_RESTRICT secret, \
4511
+ size_t nbStripes) \
4512
+{ \
4513
+ size_t n; \
4514
+ for (n = 0; n < nbStripes; n++ ) { \
4515
+ const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \
4516
+ XXH_PREFETCH(in + XXH_PREFETCH_DIST); \
4517
+ XXH3_accumulate_512_##name( \
4518
+ acc, \
4519
+ in, \
4520
+ secret + n*XXH_SECRET_CONSUME_RATE); \
4521
+ } \
4522
+}
4523
+
4524
+
4525
+XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
4526
+{
4527
+ if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
4528
+ XXH_memcpy(dst, &v64, sizeof(v64));
4529
+}
4530
+
4531
+/* Several intrinsic functions below are supposed to accept __int64 as argument,
4532
+ * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
4533
+ * However, several environments do not define __int64 type,
4534
+ * requiring a workaround.
4535
+ */
4536
+#if !defined (__VMS) \
4537
+ && (defined (__cplusplus) \
4538
+ || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
4539
+ typedef int64_t xxh_i64;
4540
+#else
4541
+ /* the following type must have a width of 64-bit */
4542
+ typedef long long xxh_i64;
4543
+#endif
4544
+
4545
+
4546
+/*
4547
+ * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
4548
+ *
4549
+ * It is a hardened version of UMAC, based off of FARSH's implementation.
4550
+ *
4551
+ * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
4552
+ * implementations, and it is ridiculously fast.
4553
+ *
4554
+ * We harden it by mixing the original input to the accumulators as well as the product.
4555
+ *
4556
+ * This means that in the (relatively likely) case of a multiply by zero, the
4557
+ * original input is preserved.
4558
+ *
4559
+ * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
4560
+ * cross-pollination, as otherwise the upper and lower halves would be
4561
+ * essentially independent.
4562
+ *
4563
+ * This doesn't matter on 64-bit hashes since they all get merged together in
4564
+ * the end, so we skip the extra step.
4565
+ *
4566
+ * Both XXH3_64bits and XXH3_128bits use this subroutine.
4567
+ */
4568
+
4569
+#if (XXH_VECTOR == XXH_AVX512) \
4570
+ || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0)
4571
+
4572
+#ifndef XXH_TARGET_AVX512
4573
+# define XXH_TARGET_AVX512 /* disable attribute target */
4574
+#endif
4575
+
4576
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
4577
+XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
4578
+ const void* XXH_RESTRICT input,
4579
+ const void* XXH_RESTRICT secret)
4580
+{
4581
+ __m512i* const xacc = (__m512i *) acc;
4582
+ XXH_ASSERT((((size_t)acc) & 63) == 0);
4583
+ XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
4584
+
4585
+ {
4586
+ /* data_vec = input[0]; */
4587
+ __m512i const data_vec = _mm512_loadu_si512 (input);
4588
+ /* key_vec = secret[0]; */
4589
+ __m512i const key_vec = _mm512_loadu_si512 (secret);
4590
+ /* data_key = data_vec ^ key_vec; */
4591
+ __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
4592
+ /* data_key_lo = data_key >> 32; */
4593
+ __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32);
4594
+ /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
4595
+ __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo);
4596
+ /* xacc[0] += swap(data_vec); */
4597
+ __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
4598
+ __m512i const sum = _mm512_add_epi64(*xacc, data_swap);
4599
+ /* xacc[0] += product; */
4600
+ *xacc = _mm512_add_epi64(product, sum);
4601
+ }
4602
+}
4603
+XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512)
4604
+
4605
+/*
4606
+ * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
4607
+ *
4608
+ * Multiplication isn't perfect, as explained by Google in HighwayHash:
4609
+ *
4610
+ * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
4611
+ * // varying degrees. In descending order of goodness, bytes
4612
+ * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
4613
+ * // As expected, the upper and lower bytes are much worse.
4614
+ *
4615
+ * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
4616
+ *
4617
+ * Since our algorithm uses a pseudorandom secret to add some variance into the
4618
+ * mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
4619
+ *
4620
+ * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
4621
+ * extraction.
4622
+ *
4623
+ * Both XXH3_64bits and XXH3_128bits use this subroutine.
4624
+ */
4625
+
4626
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
4627
+XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
4628
+{
4629
+ XXH_ASSERT((((size_t)acc) & 63) == 0);
4630
+ XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
4631
+ { __m512i* const xacc = (__m512i*) acc;
4632
+ const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);
4633
+
4634
+ /* xacc[0] ^= (xacc[0] >> 47) */
4635
+ __m512i const acc_vec = *xacc;
4636
+ __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47);
4637
+ /* xacc[0] ^= secret; */
4638
+ __m512i const key_vec = _mm512_loadu_si512 (secret);
4639
+ __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */);
4640
+
4641
+ /* xacc[0] *= XXH_PRIME32_1; */
4642
+ __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32);
4643
+ __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32);
4644
+ __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32);
4645
+ *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
4646
+ }
4647
+}
4648
+
4649
+XXH_FORCE_INLINE XXH_TARGET_AVX512 void
4650
+XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
4651
+{
4652
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
4653
+ XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
4654
+ XXH_ASSERT(((size_t)customSecret & 63) == 0);
4655
+ (void)(&XXH_writeLE64);
4656
+ { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
4657
+ __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64);
4658
+ __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos);
4659
+
4660
+ const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret);
4661
+ __m512i* const dest = ( __m512i*) customSecret;
4662
+ int i;
4663
+ XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */
4664
+ XXH_ASSERT(((size_t)dest & 63) == 0);
4665
+ for (i=0; i < nbRounds; ++i) {
4666
+ dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed);
4667
+ } }
4668
+}
4669
+
4670
+#endif
4671
+
4672
+#if (XXH_VECTOR == XXH_AVX2) \
4673
+ || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0)
4674
+
4675
+#ifndef XXH_TARGET_AVX2
4676
+# define XXH_TARGET_AVX2 /* disable attribute target */
4677
+#endif
4678
+
4679
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void
4680
+XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
4681
+ const void* XXH_RESTRICT input,
4682
+ const void* XXH_RESTRICT secret)
4683
+{
4684
+ XXH_ASSERT((((size_t)acc) & 31) == 0);
4685
+ { __m256i* const xacc = (__m256i *) acc;
4686
+ /* Unaligned. This is mainly for pointer arithmetic, and because
4687
+ * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
4688
+ const __m256i* const xinput = (const __m256i *) input;
4689
+ /* Unaligned. This is mainly for pointer arithmetic, and because
4690
+ * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
4691
+ const __m256i* const xsecret = (const __m256i *) secret;
4692
+
4693
+ size_t i;
4694
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
4695
+ /* data_vec = xinput[i]; */
4696
+ __m256i const data_vec = _mm256_loadu_si256 (xinput+i);
4697
+ /* key_vec = xsecret[i]; */
4698
+ __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
4699
+ /* data_key = data_vec ^ key_vec; */
4700
+ __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
4701
+ /* data_key_lo = data_key >> 32; */
4702
+ __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32);
4703
+ /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
4704
+ __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo);
4705
+ /* xacc[i] += swap(data_vec); */
4706
+ __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
4707
+ __m256i const sum = _mm256_add_epi64(xacc[i], data_swap);
4708
+ /* xacc[i] += product; */
4709
+ xacc[i] = _mm256_add_epi64(product, sum);
4710
+ } }
4711
+}
4712
+XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2)
4713
+
4714
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void
4715
+XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
4716
+{
4717
+ XXH_ASSERT((((size_t)acc) & 31) == 0);
4718
+ { __m256i* const xacc = (__m256i*) acc;
4719
+ /* Unaligned. This is mainly for pointer arithmetic, and because
4720
+ * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
4721
+ const __m256i* const xsecret = (const __m256i *) secret;
4722
+ const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);
4723
+
4724
+ size_t i;
4725
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
4726
+ /* xacc[i] ^= (xacc[i] >> 47) */
4727
+ __m256i const acc_vec = xacc[i];
4728
+ __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47);
4729
+ __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted);
4730
+ /* xacc[i] ^= xsecret; */
4731
+ __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
4732
+ __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
4733
+
4734
+ /* xacc[i] *= XXH_PRIME32_1; */
4735
+ __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32);
4736
+ __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32);
4737
+ __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32);
4738
+ xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
4739
+ }
4740
+ }
4741
+}
4742
+
4743
+XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
4744
+{
4745
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
4746
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
4747
+ XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
4748
+ (void)(&XXH_writeLE64);
4749
+ XXH_PREFETCH(customSecret);
4750
+ { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64);
4751
+
4752
+ const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret);
4753
+ __m256i* dest = ( __m256i*) customSecret;
4754
+
4755
+# if defined(__GNUC__) || defined(__clang__)
4756
+ /*
4757
+ * On GCC & Clang, marking 'dest' as modified will cause the compiler:
4758
+ * - do not extract the secret from sse registers in the internal loop
4759
+ * - use less common registers, and avoid pushing these reg into stack
4760
+ */
4761
+ XXH_COMPILER_GUARD(dest);
4762
+# endif
4763
+ XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */
4764
+ XXH_ASSERT(((size_t)dest & 31) == 0);
4765
+
4766
+ /* GCC -O2 need unroll loop manually */
4767
+ dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed);
4768
+ dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed);
4769
+ dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed);
4770
+ dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed);
4771
+ dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed);
4772
+ dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed);
4773
+ }
4774
+}
4775
+
4776
+#endif
4777
+
4778
+/* x86dispatch always generates SSE2 */
4779
+#if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH)
4780
+
4781
+#ifndef XXH_TARGET_SSE2
4782
+# define XXH_TARGET_SSE2 /* disable attribute target */
4783
+#endif
4784
+
4785
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void
4786
+XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
4787
+ const void* XXH_RESTRICT input,
4788
+ const void* XXH_RESTRICT secret)
4789
+{
4790
+ /* SSE2 is just a half-scale version of the AVX2 version. */
4791
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
4792
+ { __m128i* const xacc = (__m128i *) acc;
4793
+ /* Unaligned. This is mainly for pointer arithmetic, and because
4794
+ * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
4795
+ const __m128i* const xinput = (const __m128i *) input;
4796
+ /* Unaligned. This is mainly for pointer arithmetic, and because
4797
+ * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
4798
+ const __m128i* const xsecret = (const __m128i *) secret;
4799
+
4800
+ size_t i;
4801
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
4802
+ /* data_vec = xinput[i]; */
4803
+ __m128i const data_vec = _mm_loadu_si128 (xinput+i);
4804
+ /* key_vec = xsecret[i]; */
4805
+ __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
4806
+ /* data_key = data_vec ^ key_vec; */
4807
+ __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
4808
+ /* data_key_lo = data_key >> 32; */
4809
+ __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
4810
+ /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
4811
+ __m128i const product = _mm_mul_epu32 (data_key, data_key_lo);
4812
+ /* xacc[i] += swap(data_vec); */
4813
+ __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
4814
+ __m128i const sum = _mm_add_epi64(xacc[i], data_swap);
4815
+ /* xacc[i] += product; */
4816
+ xacc[i] = _mm_add_epi64(product, sum);
4817
+ } }
4818
+}
4819
+XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2)
4820
+
4821
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void
4822
+XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
4823
+{
4824
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
4825
+ { __m128i* const xacc = (__m128i*) acc;
4826
+ /* Unaligned. This is mainly for pointer arithmetic, and because
4827
+ * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
4828
+ const __m128i* const xsecret = (const __m128i *) secret;
4829
+ const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);
4830
+
4831
+ size_t i;
4832
+ for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
4833
+ /* xacc[i] ^= (xacc[i] >> 47) */
4834
+ __m128i const acc_vec = xacc[i];
4835
+ __m128i const shifted = _mm_srli_epi64 (acc_vec, 47);
4836
+ __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted);
4837
+ /* xacc[i] ^= xsecret[i]; */
4838
+ __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
4839
+ __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
4840
+
4841
+ /* xacc[i] *= XXH_PRIME32_1; */
4842
+ __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
4843
+ __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32);
4844
+ __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32);
4845
+ xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
4846
+ }
4847
+ }
4848
+}
4849
+
4850
+XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
4851
+{
4852
+ XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
4853
+ (void)(&XXH_writeLE64);
4854
+ { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);
4855
+
4856
+# if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900
4857
+ /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */
4858
+ XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) };
4859
+ __m128i const seed = _mm_load_si128((__m128i const*)seed64x2);
4860
+# else
4861
+ __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64);
4862
+# endif
4863
+ int i;
4864
+
4865
+ const void* const src16 = XXH3_kSecret;
4866
+ __m128i* dst16 = (__m128i*) customSecret;
4867
+# if defined(__GNUC__) || defined(__clang__)
4868
+ /*
4869
+ * On GCC & Clang, marking 'dest' as modified will cause the compiler:
4870
+ * - do not extract the secret from sse registers in the internal loop
4871
+ * - use less common registers, and avoid pushing these reg into stack
4872
+ */
4873
+ XXH_COMPILER_GUARD(dst16);
4874
+# endif
4875
+ XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */
4876
+ XXH_ASSERT(((size_t)dst16 & 15) == 0);
4877
+
4878
+ for (i=0; i < nbRounds; ++i) {
4879
+ dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed);
4880
+ } }
4881
+}
4882
+
4883
+#endif
4884
+
4885
+#if (XXH_VECTOR == XXH_NEON)
4886
+
4887
+/* forward declarations for the scalar routines */
4888
+XXH_FORCE_INLINE void
4889
+XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input,
4890
+ void const* XXH_RESTRICT secret, size_t lane);
4891
+
4892
+XXH_FORCE_INLINE void
4893
+XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
4894
+ void const* XXH_RESTRICT secret, size_t lane);
4895
+
4896
+/*!
4897
+ * @internal
4898
+ * @brief The bulk processing loop for NEON and WASM SIMD128.
4899
+ *
4900
+ * The NEON code path is actually partially scalar when running on AArch64. This
4901
+ * is to optimize the pipelining and can have up to 15% speedup depending on the
4902
+ * CPU, and it also mitigates some GCC codegen issues.
4903
+ *
4904
+ * @see XXH3_NEON_LANES for configuring this and details about this optimization.
4905
+ *
4906
+ * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit
4907
+ * integers instead of the other platforms which mask full 64-bit vectors,
4908
+ * so the setup is more complicated than just shifting right.
4909
+ *
4910
+ * Additionally, there is an optimization for 4 lanes at once noted below.
4911
+ *
4912
+ * Since, as stated, the most optimal amount of lanes for Cortexes is 6,
4913
+ * there needs to be *three* versions of the accumulate operation used
4914
+ * for the remaining 2 lanes.
4915
+ *
4916
+ * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap
4917
+ * nearly perfectly.
4918
+ */
4919
+
4920
+XXH_FORCE_INLINE void
4921
+XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
4922
+ const void* XXH_RESTRICT input,
4923
+ const void* XXH_RESTRICT secret)
4924
+{
4925
+ XXH_ASSERT((((size_t)acc) & 15) == 0);
4926
+ XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0);
4927
+ { /* GCC for darwin arm64 does not like aliasing here */
4928
+ xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc;
4929
+ /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
4930
+ uint8_t const* xinput = (const uint8_t *) input;
4931
+ uint8_t const* xsecret = (const uint8_t *) secret;
4932
+
4933
+ size_t i;
4934
+#ifdef __wasm_simd128__
4935
+ /*
4936
+ * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret
4937
+ * is constant propagated, which results in it converting it to this
4938
+ * inside the loop:
4939
+ *
4940
+ * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0)
4941
+ * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0)
4942
+ * ...
4943
+ *
4944
+ * This requires a full 32-bit address immediate (and therefore a 6 byte
4945
+ * instruction) as well as an add for each offset.
4946
+ *
4947
+ * Putting an asm guard prevents it from folding (at the cost of losing
4948
+ * the alignment hint), and uses the free offset in `v128.load` instead
4949
+ * of adding secret_offset each time which overall reduces code size by
4950
+ * about a kilobyte and improves performance.
4951
+ */
4952
+ XXH_COMPILER_GUARD(xsecret);
4953
+#endif
4954
+ /* Scalar lanes use the normal scalarRound routine */
4955
+ for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
4956
+ XXH3_scalarRound(acc, input, secret, i);
4957
+ }
4958
+ i = 0;
4959
+ /* 4 NEON lanes at a time. */
4960
+ for (; i+1 < XXH3_NEON_LANES / 2; i+=2) {
4961
+ /* data_vec = xinput[i]; */
4962
+ uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16));
4963
+ uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16));
4964
+ /* key_vec = xsecret[i]; */
4965
+ uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16));
4966
+ uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16));
4967
+ /* data_swap = swap(data_vec) */
4968
+ uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1);
4969
+ uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1);
4970
+ /* data_key = data_vec ^ key_vec; */
4971
+ uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1);
4972
+ uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2);
4973
+
4974
+ /*
4975
+ * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a
4976
+ * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to
4977
+ * get one vector with the low 32 bits of each lane, and one vector
4978
+ * with the high 32 bits of each lane.
4979
+ *
4980
+ * The intrinsic returns a double vector because the original ARMv7-a
4981
+ * instruction modified both arguments in place. AArch64 and SIMD128 emit
4982
+ * two instructions from this intrinsic.
4983
+ *
4984
+ * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ]
4985
+ * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ]
4986
+ */
4987
+ uint32x4x2_t unzipped = vuzpq_u32(
4988
+ vreinterpretq_u32_u64(data_key_1),
4989
+ vreinterpretq_u32_u64(data_key_2)
4990
+ );
4991
+ /* data_key_lo = data_key & 0xFFFFFFFF */
4992
+ uint32x4_t data_key_lo = unzipped.val[0];
4993
+ /* data_key_hi = data_key >> 32 */
4994
+ uint32x4_t data_key_hi = unzipped.val[1];
4995
+ /*
4996
+ * Then, we can split the vectors horizontally and multiply which, as for most
4997
+ * widening intrinsics, have a variant that works on both high half vectors
4998
+ * for free on AArch64. A similar instruction is available on SIMD128.
4999
+ *
This file is too large to show in full.
streaming/sender.c
+3
-1
@@ -981,7 +981,9 @@ void execute_commands(struct sender_state *s) {
981
982
int code = rrd_call_function_async(s->host, wb, timeout, function, stream_execute_function_callback, tmp);
983
if(code != HTTP_RESP_OK) {
984
- rrd_call_function_error(wb, "Failed to route request to collector", code);
984
+ if (!buffer_strlen(wb))
985
+ rrd_call_function_error(wb, "Failed to route request to collector", code);
986
+
987
stream_execute_function_callback(wb, code, tmp);
988
}
989
}
web/api/formatters/rrd2json.c
+1
-1
@@ -170,7 +170,7 @@ int data_query_execute(ONEWAYALLOC *owa, BUFFER *wb, QUERY_TARGET *qt, time_t *l
170
171
if (r->view.flags & RRDR_RESULT_FLAG_CANCEL) {
172
rrdr_free(owa, r);
173
- return HTTP_RESP_BACKEND_FETCH_FAILED;
173
+ return HTTP_RESP_CLIENT_CLOSED_REQUEST;
174
}
175
176
if(r->view.flags & RRDR_RESULT_FLAG_RELATIVE)
web/api/queries/weights.c
+1
-1
@@ -1913,7 +1913,7 @@ int web_api_v12_weights(BUFFER *wb, QUERY_WEIGHTS_REQUEST *qwr) {
1913
1914
if(qwd.interrupted) {
1915
error = "interrupted";
1916
- resp = HTTP_RESP_BACKEND_FETCH_FAILED;
1916
+ resp = HTTP_RESP_CLIENT_CLOSED_REQUEST;
1917
goto cleanup;
1918
}
1919
web/api/web_api.c
+1
-1
@@ -142,7 +142,7 @@ RRDCONTEXT_TO_JSON_OPTIONS rrdcontext_to_json_parse_options(char *o) {
142
143
int web_client_api_request_weights(RRDHOST *host, struct web_client *w, char *url, WEIGHTS_METHOD method, WEIGHTS_FORMAT format, size_t api_version) {
144
if (!netdata_ready)
145
- return HTTP_RESP_BACKEND_FETCH_FAILED;
145
+ return HTTP_RESP_SERVICE_UNAVAILABLE;
146
147
time_t baseline_after = 0, baseline_before = 0, after = 0, before = 0;
148
size_t points = 0;
web/api/web_api_v1.c
+7
-7
@@ -1313,7 +1313,7 @@ int web_client_api_request_v1_ml_info(RRDHOST *host, struct web_client *w, char
1313
(void) url;
1314
1315
if (!netdata_ready)
1316
- return HTTP_RESP_BACKEND_FETCH_FAILED;
1316
+ return HTTP_RESP_SERVICE_UNAVAILABLE;
1317
1318
BUFFER *wb = w->response.data;
1319
buffer_flush(wb);
@@ -1332,7 +1332,7 @@ int web_client_api_request_v1_ml_models(RRDHOST *host, struct web_client *w, cha
1332
(void) url;
1333
1334
if (!netdata_ready)
1335
- return HTTP_RESP_BACKEND_FETCH_FAILED;
1335
+ return HTTP_RESP_SERVICE_UNAVAILABLE;
1336
1337
BUFFER *wb = w->response.data;
1338
buffer_flush(wb);
@@ -1346,7 +1346,7 @@ int web_client_api_request_v1_ml_models(RRDHOST *host, struct web_client *w, cha
1346
1347
inline int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, char *url) {
1348
(void)url;
1349
- if (!netdata_ready) return HTTP_RESP_BACKEND_FETCH_FAILED;
1349
+ if (!netdata_ready) return HTTP_RESP_SERVICE_UNAVAILABLE;
1350
BUFFER *wb = w->response.data;
1351
buffer_flush(wb);
1352
wb->content_type = CT_APPLICATION_JSON;
@@ -1360,7 +1360,7 @@ inline int web_client_api_request_v1_info(RRDHOST *host, struct web_client *w, c
1360
static int web_client_api_request_v1_aclk_state(RRDHOST *host, struct web_client *w, char *url) {
1361
UNUSED(url);
1362
UNUSED(host);
1363
- if (!netdata_ready) return HTTP_RESP_BACKEND_FETCH_FAILED;
1363
+ if (!netdata_ready) return HTTP_RESP_SERVICE_UNAVAILABLE;
1364
1365
BUFFER *wb = w->response.data;
1366
buffer_flush(wb);
@@ -1386,7 +1386,7 @@ int web_client_api_request_v1_weights(RRDHOST *host, struct web_client *w, char
1386
1387
int web_client_api_request_v1_function(RRDHOST *host, struct web_client *w, char *url) {
1388
if (!netdata_ready)
1389
- return HTTP_RESP_BACKEND_FETCH_FAILED;
1389
+ return HTTP_RESP_SERVICE_UNAVAILABLE;
1390
1391
int timeout = 0;
1392
const char *function = NULL;
@@ -1417,7 +1417,7 @@ int web_client_api_request_v1_function(RRDHOST *host, struct web_client *w, char
1417
1418
int web_client_api_request_v1_functions(RRDHOST *host, struct web_client *w, char *url __maybe_unused) {
1419
if (!netdata_ready)
1420
- return HTTP_RESP_BACKEND_FETCH_FAILED;
1420
+ return HTTP_RESP_SERVICE_UNAVAILABLE;
1421
1422
BUFFER *wb = w->response.data;
1423
buffer_flush(wb);
@@ -1500,7 +1500,7 @@ static void web_client_api_v1_dbengine_stats_for_tier(BUFFER *wb, size_t tier) {
1500
}
1501
int web_client_api_request_v1_dbengine_stats(RRDHOST *host __maybe_unused, struct web_client *w, char *url __maybe_unused) {
1502
if (!netdata_ready)
1503
- return HTTP_RESP_BACKEND_FETCH_FAILED;
1503
+ return HTTP_RESP_SERVICE_UNAVAILABLE;
1504
1505
BUFFER *wb = w->response.data;
1506
buffer_flush(wb);
web/server/web_client.c
+3
-1
@@ -928,6 +928,8 @@ const char *web_response_code_to_string(int code) {
928
return "Request Header Fields Too Large";
929
case 451:
930
return "Unavailable For Legal Reasons";
931
+ case 499: // nginx's extension to the standard
932
+ return "Client Closed Request";
933
934
case 500:
935
return "Internal Server Error";
@@ -2437,7 +2439,7 @@ inline bool web_client_timeout_checkpoint_and_check(struct web_client *w, usec_t
2439
if (since_reception_ut >= w->timings.timeout_ut) {
2440
buffer_flush(w->response.data);
2441
buffer_strcat(w->response.data, "Query timeout exceeded");
2440
- w->response.code = HTTP_RESP_BACKEND_FETCH_FAILED;
2442
+ w->response.code = HTTP_RESP_GATEWAY_TIMEOUT;
2443
return true;
2444
}
2445