@samitouri / QOS-React-2 / commits / cfbf115b26

Add hardware-counter.cpp and deps from HHVM

Ben Alpert committed Sep 18, 2015 at 21:43 UTC cfbf115b2646a08c1eb2cb50c4ee8cd7602f3624
5 files changed +1659
scripts/perf-counters/src/hardware-counter.cpp new
+481
@@ -0,0 +1,481 @@
1 +/*
2 + +----------------------------------------------------------------------+
3 + | HipHop for PHP |
4 + +----------------------------------------------------------------------+
5 + | Copyright (c) 2010-2015 Facebook, Inc. (http://www.facebook.com) |
6 + +----------------------------------------------------------------------+
7 + | This source file is subject to version 3.01 of the PHP license, |
8 + | that is bundled with this package in the file LICENSE, and is |
9 + | available through the world-wide-web at the following url: |
10 + | http://www.php.net/license/3_01.txt |
11 + | If you did not receive a copy of the PHP license and are unable to |
12 + | obtain it through the world-wide-web, please send a note to |
13 + | license@php.net so we can mail you a copy immediately. |
14 + +----------------------------------------------------------------------+
15 +*/
16 +
17 +#include "hphp/util/hardware-counter.h"
18 +
19 +#ifndef NO_HARDWARE_COUNTERS
20 +
21 +#include <folly/ScopeGuard.h>
22 +
23 +#include "hphp/util/logger.h"
24 +
25 +#define _GNU_SOURCE 1
26 +#include <stdio.h>
27 +#include <stdlib.h>
28 +#include <string.h>
29 +#include <unistd.h>
30 +#include <fcntl.h>
31 +#include <errno.h>
32 +#include <assert.h>
33 +#include <sys/mman.h>
34 +#include <sys/ioctl.h>
35 +#include <asm/unistd.h>
36 +#include <sys/prctl.h>
37 +#include <linux/perf_event.h>
38 +#include <folly/String.h>
39 +#include <folly/Memory.h>
40 +
41 +namespace HPHP {
42 +///////////////////////////////////////////////////////////////////////////////
43 +
44 +IMPLEMENT_THREAD_LOCAL_NO_CHECK(HardwareCounter,
45 + HardwareCounter::s_counter);
46 +
47 +static bool s_recordSubprocessTimes = false;
48 +static bool s_profileHWEnable;
49 +static std::string s_profileHWEvents;
50 +
51 +static inline bool useCounters() {
52 +#ifdef VALGRIND
53 + return false;
54 +#else
55 + return s_profileHWEnable;
56 +#endif
57 +}
58 +
59 +class HardwareCounterImpl {
60 +public:
61 + HardwareCounterImpl(int type, unsigned long config,
62 + const char* desc = nullptr)
63 + : m_desc(desc ? desc : ""), m_err(0), m_fd(-1), inited(false) {
64 + memset (&pe, 0, sizeof (struct perf_event_attr));
65 + pe.type = type;
66 + pe.size = sizeof (struct perf_event_attr);
67 + pe.config = config;
68 + pe.inherit = s_recordSubprocessTimes;
69 + pe.disabled = 1;
70 + pe.pinned = 0;
71 + pe.exclude_kernel = 0;
72 + pe.exclude_hv = 1;
73 + pe.read_format =
74 + PERF_FORMAT_TOTAL_TIME_ENABLED|PERF_FORMAT_TOTAL_TIME_RUNNING;
75 + }
76 +
77 + ~HardwareCounterImpl() {
78 + close();
79 + }
80 +
81 + void init_if_not() {
82 + /*
83 + * perf_event_open(struct perf_event_attr *hw_event_uptr, pid_t pid,
84 + * int cpu, int group_fd, unsigned long flags)
85 + */
86 + if (inited) return;
87 + inited = true;
88 + m_fd = syscall(__NR_perf_event_open, &pe, 0, -1, -1, 0);
89 + if (m_fd < 0) {
90 + Logger::Verbose("perf_event_open failed with: %s",
91 + folly::errnoStr(errno).c_str());
92 + m_err = -1;
93 + return;
94 + }
95 + if (ioctl(m_fd, PERF_EVENT_IOC_ENABLE, 0) < 0) {
96 + Logger::Warning("perf_event failed to enable: %s",
97 + folly::errnoStr(errno).c_str());
98 + close();
99 + m_err = -1;
100 + return;
101 + }
102 + reset();
103 + }
104 +
105 + int64_t read() {
106 + uint64_t values[3];
107 + if (readRaw(values)) {
108 + if (!values[2]) return 0;
109 + int64_t value = (double)values[0] * values[1] / values[2];
110 + return value + extra;
111 + }
112 + return 0;
113 + }
114 +
115 + void incCount(int64_t amount) {
116 + extra += amount;
117 + }
118 +
119 + bool readRaw(uint64_t* values) {
120 + if (m_err || !useCounters()) return false;
121 + init_if_not();
122 +
123 + if (m_fd > 0) {
124 + /*
125 + * read the count + scaling values
126 + *
127 + * It is not necessary to stop an event to read its value
128 + */
129 + auto ret = ::read(m_fd, values, sizeof(*values) * 3);
130 + if (ret == sizeof(*values) * 3) {
131 + values[0] -= reset_values[0];
132 + values[1] -= reset_values[1];
133 + values[2] -= reset_values[2];
134 + return true;
135 + }
136 + }
137 + return false;
138 + }
139 +
140 + void reset() {
141 + if (m_err || !useCounters()) return;
142 + init_if_not();
143 + extra = 0;
144 + if (m_fd > 0) {
145 + if (ioctl (m_fd, PERF_EVENT_IOC_RESET, 0) < 0) {
146 + Logger::Warning("perf_event failed to reset with: %s",
147 + folly::errnoStr(errno).c_str());
148 + m_err = -1;
149 + return;
150 + }
151 + auto ret = ::read(m_fd, reset_values, sizeof(reset_values));
152 + if (ret != sizeof(reset_values)) {
153 + Logger::Warning("perf_event failed to reset with: %s",
154 + folly::errnoStr(errno).c_str());
155 + m_err = -1;
156 + return;
157 + }
158 + }
159 + }
160 +
161 +public:
162 + std::string m_desc;
163 + int m_err;
164 +private:
165 + int m_fd;
166 + struct perf_event_attr pe;
167 + bool inited;
168 + uint64_t reset_values[3];
169 + uint64_t extra{0};
170 +
171 + void close() {
172 + if (m_fd > 0) {
173 + ::close(m_fd);
174 + m_fd = -1;
175 + }
176 + }
177 +};
178 +
179 +class InstructionCounter : public HardwareCounterImpl {
180 +public:
181 + InstructionCounter() :
182 + HardwareCounterImpl(PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS) {}
183 +};
184 +
185 +class LoadCounter : public HardwareCounterImpl {
186 +public:
187 + LoadCounter() :
188 + HardwareCounterImpl(PERF_TYPE_HW_CACHE,
189 + (PERF_COUNT_HW_CACHE_L1D | ((PERF_COUNT_HW_CACHE_OP_READ) << 8))) {}
190 +};
191 +
192 +class StoreCounter : public HardwareCounterImpl {
193 +public:
194 + StoreCounter() :
195 + HardwareCounterImpl(PERF_TYPE_HW_CACHE,
196 + PERF_COUNT_HW_CACHE_L1D | ((PERF_COUNT_HW_CACHE_OP_WRITE) << 8)) {}
197 +};
198 +
199 +HardwareCounter::HardwareCounter()
200 + : m_countersSet(false) {
201 + m_instructionCounter.reset(new InstructionCounter());
202 + if (s_profileHWEvents.empty()) {
203 + m_loadCounter.reset(new LoadCounter());
204 + m_storeCounter.reset(new StoreCounter());
205 + } else {
206 + m_countersSet = true;
207 + setPerfEvents(s_profileHWEvents);
208 + }
209 +}
210 +
211 +HardwareCounter::~HardwareCounter() {
212 +}
213 +
214 +void HardwareCounter::Init(bool enable, const std::string& events,
215 + bool subProc) {
216 + s_profileHWEnable = enable;
217 + s_profileHWEvents = events;
218 + s_recordSubprocessTimes = subProc;
219 +}
220 +
221 +void HardwareCounter::Reset() {
222 + s_counter->reset();
223 +}
224 +
225 +void HardwareCounter::reset() {
226 + m_instructionCounter->reset();
227 + if (!m_countersSet) {
228 + m_storeCounter->reset();
229 + m_loadCounter->reset();
230 + }
231 + for (unsigned i = 0; i < m_counters.size(); i++) {
232 + m_counters[i]->reset();
233 + }
234 +}
235 +
236 +int64_t HardwareCounter::GetInstructionCount() {
237 + return s_counter->getInstructionCount();
238 +}
239 +
240 +int64_t HardwareCounter::getInstructionCount() {
241 + return m_instructionCounter->read();
242 +}
243 +
244 +int64_t HardwareCounter::GetLoadCount() {
245 + return s_counter->getLoadCount();
246 +}
247 +
248 +int64_t HardwareCounter::getLoadCount() {
249 + return m_loadCounter->read();
250 +}
251 +
252 +int64_t HardwareCounter::GetStoreCount() {
253 + return s_counter->getStoreCount();
254 +}
255 +
256 +int64_t HardwareCounter::getStoreCount() {
257 + return m_storeCounter->read();
258 +}
259 +
260 +void HardwareCounter::IncInstructionCount(int64_t amount) {
261 + s_counter->m_instructionCounter->incCount(amount);
262 +}
263 +
264 +void HardwareCounter::IncLoadCount(int64_t amount) {
265 + if (!s_counter->m_countersSet) {
266 + s_counter->m_loadCounter->incCount(amount);
267 + }
268 +}
269 +
270 +void HardwareCounter::IncStoreCount(int64_t amount) {
271 + if (!s_counter->m_countersSet) {
272 + s_counter->m_storeCounter->incCount(amount);
273 + }
274 +}
275 +
276 +struct PerfTable perfTable[] = {
277 + /* PERF_TYPE_HARDWARE events */
278 +#define PC(n) PERF_TYPE_HARDWARE, PERF_COUNT_HW_ ## n
279 + { "cpu-cycles", PC(CPU_CYCLES) },
280 + { "cycles", PC(CPU_CYCLES) },
281 + { "instructions", PC(INSTRUCTIONS) },
282 + { "cache-references", PC(CACHE_REFERENCES) },
283 + { "cache-misses", PC(CACHE_MISSES) },
284 + { "branch-instructions", PC(BRANCH_INSTRUCTIONS) },
285 + { "branches", PC(BRANCH_INSTRUCTIONS) },
286 + { "branch-misses", PC(BRANCH_MISSES) },
287 + { "bus-cycles", PC(BUS_CYCLES) },
288 + { "stalled-cycles-frontend", PC(STALLED_CYCLES_FRONTEND) },
289 + { "stalled-cycles-backend", PC(STALLED_CYCLES_BACKEND) },
290 +
291 + /* PERF_TYPE_HW_CACHE hw_cache_id */
292 +#define PCC(n) PERF_TYPE_HW_CACHE, PERF_COUNT_HW_CACHE_ ## n
293 + { "L1-dcache-", PCC(L1D) },
294 + { "L1-icache-", PCC(L1I) },
295 + { "LLC-", PCC(LL) },
296 + { "dTLB-", PCC(DTLB) },
297 + { "iTLB-", PCC(ITLB) },
298 + { "branch-", PCC(BPU) },
299 +
300 + /* PERF_TYPE_HW_CACHE hw_cache_op, hw_cache_result */
301 +#define PCCO(n, m) PERF_TYPE_HW_CACHE, \
302 + ((PERF_COUNT_HW_CACHE_OP_ ## n) << 8 | \
303 + (PERF_COUNT_HW_CACHE_RESULT_ ## m) << 16)
304 + { "loads", PCCO(READ, ACCESS) },
305 + { "load-misses", PCCO(READ, MISS) },
306 + { "stores", PCCO(WRITE, ACCESS) },
307 + { "store-misses", PCCO(WRITE, MISS) },
308 + { "prefetches", PCCO(PREFETCH, ACCESS) },
309 + { "prefetch-misses", PCCO(PREFETCH, MISS) }
310 +};
311 +
312 +static int findEvent(const char *event, struct PerfTable *t,
313 + int len, int *match_len) {
314 + int i;
315 +
316 + for (i = 0; i < len; i++) {
317 + if (!strncmp(event, t[i].name, strlen(t[i].name))) {
318 + *match_len = strlen(t[i].name);
319 + return i;
320 + }
321 + }
322 + return -1;
323 +}
324 +
325 +#define CPUID_STEPPING(x) ((x) & 0xf)
326 +#define CPUID_MODEL(x) (((x) & 0xf0) >> 4)
327 +#define CPUID_FAMILY(x) (((x) & 0xf00) >> 8)
328 +#define CPUID_TYPE(x) (((x) & 0x3000) >> 12)
329 +
330 +// hack to get LLC counters on perflab frc machines
331 +static bool isIntelE5_2670() {
332 +#ifdef __x86_64__
333 + unsigned long x;
334 + asm volatile ("cpuid" : "=a"(x): "a"(1) : "ebx", "ecx", "edx");
335 + return CPUID_STEPPING(x) == 6 && CPUID_MODEL(x) == 0xd
336 + && CPUID_FAMILY(x) == 6 && CPUID_TYPE(x) == 0;
337 +#else
338 + return false;
339 +#endif
340 +}
341 +
342 +static void checkLLCHack(const char* event, uint32_t& type, uint64_t& config) {
343 + if (!strncmp(event, "LLC-load", 8) && isIntelE5_2670()) {
344 + type = PERF_TYPE_RAW;
345 + if (!strncmp(&event[4], "loads", 5)) {
346 + config = 0x534f2e;
347 + } else if (!strncmp(&event[4], "load-misses", 11)) {
348 + config = 0x53412e;
349 + }
350 + }
351 +}
352 +
353 +bool HardwareCounter::addPerfEvent(const char* event) {
354 + uint32_t type = 0;
355 + uint64_t config = 0;
356 + int i, match_len;
357 + bool found = false;
358 + const char* ev = event;
359 +
360 + while ((i = findEvent(ev, perfTable,
361 + sizeof(perfTable)/sizeof(struct PerfTable),
362 + &match_len))
363 + != -1) {
364 + if (!found) {
365 + found = true;
366 + type = perfTable[i].type;
367 + } else if (type != perfTable[i].type) {
368 + Logger::Warning("failed to find perf event: %s", event);
369 + return false;
370 + }
371 + config |= perfTable[i].config;
372 + ev = &ev[match_len];
373 + }
374 +
375 + checkLLCHack(event, type, config);
376 +
377 + // Check if we have a raw spec.
378 + if (!found && event[0] == 'r' && event[1] != 0) {
379 + config = strtoull(event + 1, const_cast<char**>(&ev), 16);
380 + if (*ev == 0) {
381 + found = true;
382 + type = PERF_TYPE_RAW;
383 + }
384 + }
385 +
386 + if (!found || *ev) {
387 + Logger::Warning("failed to find perf event: %s", event);
388 + return false;
389 + }
390 + auto hwc = folly::make_unique<HardwareCounterImpl>(type, config, event);
391 + if (hwc->m_err) {
392 + Logger::Warning("failed to set perf event: %s", event);
393 + return false;
394 + }
395 + m_counters.emplace_back(std::move(hwc));
396 + if (!m_countersSet) {
397 + // reset load and store counters. This is because
398 + // perf does not seem to handle more than three counters
399 + // very well.
400 + m_loadCounter.reset();
401 + m_storeCounter.reset();
402 + m_countersSet = true;
403 + }
404 + return true;
405 +}
406 +
407 +bool HardwareCounter::eventExists(const char *event) {
408 + // hopefully m_counters set is small, so a linear scan does not hurt
409 + for(unsigned i = 0; i < m_counters.size(); i++) {
410 + if (!strcmp(event, m_counters[i]->m_desc.c_str())) {
411 + return true;
412 + }
413 + }
414 + return false;
415 +}
416 +
417 +bool HardwareCounter::setPerfEvents(folly::StringPiece sevents) {
418 + // Make a copy of the string for use with strtok.
419 + auto const sevents_buf = static_cast<char*>(malloc(sevents.size() + 1));
420 + SCOPE_EXIT { free(sevents_buf); };
421 + memcpy(sevents_buf, sevents.data(), sevents.size());
422 + sevents_buf[sevents.size()] = '\0';
423 +
424 + char* strtok_buf = nullptr;
425 + char* s = strtok_r(sevents_buf, ",", &strtok_buf);
426 + while (s) {
427 + if (!eventExists(s) && !addPerfEvent(s)) {
428 + return false;
429 + }
430 + s = strtok_r(nullptr, ",", &strtok_buf);
431 + }
432 + return true;
433 +}
434 +
435 +bool HardwareCounter::SetPerfEvents(folly::StringPiece events) {
436 + return s_counter->setPerfEvents(events);
437 +}
438 +
439 +void HardwareCounter::clearPerfEvents() {
440 + m_counters.clear();
441 +}
442 +
443 +void HardwareCounter::ClearPerfEvents() {
444 + s_counter->clearPerfEvents();
445 +}
446 +
447 +const std::string
448 + s_instructions("instructions"),
449 + s_loads("loads"),
450 + s_stores("stores");
451 +
452 +void HardwareCounter::getPerfEvents(PerfEventCallback f, void* data) {
453 + f(s_instructions, getInstructionCount(), data);
454 + if (!m_countersSet) {
455 + f(s_loads, getLoadCount(), data);
456 + f(s_stores, getStoreCount(), data);
457 + }
458 + for (unsigned i = 0; i < m_counters.size(); i++) {
459 + f(m_counters[i]->m_desc, m_counters[i]->read(), data);
460 + }
461 +}
462 +
463 +void HardwareCounter::GetPerfEvents(PerfEventCallback f, void* data) {
464 + s_counter->getPerfEvents(f, data);
465 +}
466 +
467 +///////////////////////////////////////////////////////////////////////////////
468 +}
469 +
470 +
471 +#else // NO_HARDWARE_COUNTERS
472 +
473 +namespace HPHP {
474 +///////////////////////////////////////////////////////////////////////////////
475 +
476 +HardwareCounter HardwareCounter::s_counter;
477 +
478 +///////////////////////////////////////////////////////////////////////////////
479 +}
480 +
481 +#endif // NO_HARDWARE_COUNTERS
scripts/perf-counters/src/hardware-counter.h new
+118
@@ -0,0 +1,118 @@
1 +/*
2 + +----------------------------------------------------------------------+
3 + | HipHop for PHP |
4 + +----------------------------------------------------------------------+
5 + | Copyright (c) 2010-2015 Facebook, Inc. (http://www.facebook.com) |
6 + +----------------------------------------------------------------------+
7 + | This source file is subject to version 3.01 of the PHP license, |
8 + | that is bundled with this package in the file LICENSE, and is |
9 + | available through the world-wide-web at the following url: |
10 + | http://www.php.net/license/3_01.txt |
11 + | If you did not receive a copy of the PHP license and are unable to |
12 + | obtain it through the world-wide-web, please send a note to |
13 + | license@php.net so we can mail you a copy immediately. |
14 + +----------------------------------------------------------------------+
15 +*/
16 +
17 +#ifndef incl_HPHP_UTIL_HARDWARE_COUNTER_H_
18 +#define incl_HPHP_UTIL_HARDWARE_COUNTER_H_
19 +
20 +#include "hphp/util/thread-local.h"
21 +
22 +#include <folly/Range.h>
23 +
24 +#include <cstdint>
25 +#include <vector>
26 +
27 +namespace HPHP {
28 +///////////////////////////////////////////////////////////////////////////////
29 +
30 +#ifndef NO_HARDWARE_COUNTERS
31 +
32 +class InstructionCounter;
33 +class LoadCounter;
34 +class StoreCounter;
35 +
36 +struct PerfTable {
37 + const char* name;
38 + uint32_t type;
39 + uint64_t config;
40 +};
41 +
42 +class HardwareCounterImpl;
43 +
44 +class HardwareCounter {
45 +public:
46 + HardwareCounter();
47 + ~HardwareCounter();
48 +
49 + static void Reset();
50 + static int64_t GetInstructionCount();
51 + static int64_t GetLoadCount();
52 + static int64_t GetStoreCount();
53 + static bool SetPerfEvents(folly::StringPiece events);
54 + static void IncInstructionCount(int64_t amount);
55 + static void IncLoadCount(int64_t amount);
56 + static void IncStoreCount(int64_t amount);
57 +
58 + typedef void (*PerfEventCallback)(const std::string&, int64_t, void*);
59 + static void GetPerfEvents(PerfEventCallback f, void* data);
60 + static void ClearPerfEvents();
61 + static void Init(bool enable, const std::string& events, bool subProc);
62 + static DECLARE_THREAD_LOCAL_NO_CHECK(HardwareCounter, s_counter);
63 + bool m_countersSet{false};
64 +private:
65 + void reset();
66 + int64_t getInstructionCount();
67 + int64_t getLoadCount();
68 + int64_t getStoreCount();
69 + bool eventExists(const char* event);
70 + bool addPerfEvent(const char* event);
71 + bool setPerfEvents(folly::StringPiece events);
72 + void getPerfEvents(PerfEventCallback f, void* data);
73 + void clearPerfEvents();
74 +
75 + std::unique_ptr<InstructionCounter> m_instructionCounter;
76 + std::unique_ptr<LoadCounter> m_loadCounter;
77 + std::unique_ptr<StoreCounter> m_storeCounter;
78 + std::vector<std::unique_ptr<HardwareCounterImpl>> m_counters;
79 +};
80 +
81 +#else // NO_HARDWARE_COUNTERS
82 +
83 +/* Stub implementation for platforms without hardware counters (non-linux)
84 + * This mock class pretends to track performance events, but just returns
85 + * static values, so it doesn't even need to worry about thread safety
86 + * for the one static instance of itself.
87 + */
88 +class HardwareCounter {
89 +public:
90 + HardwareCounter() : m_countersSet(false) { }
91 + ~HardwareCounter() { }
92 +
93 + static void Reset() { }
94 + static int64_t GetInstructionCount() { return 0; }
95 + static int64_t GetLoadCount() { return 0; }
96 + static int64_t GetStoreCount() { return 0; }
97 + static bool SetPerfEvents(folly::StringPiece events) { return false; }
98 + static void IncInstructionCount(int64_t amount) {}
99 + static void IncLoadCount(int64_t amount) {}
100 + static void IncStoreCount(int64_t amount) {}
101 + typedef void (*PerfEventCallback)(const std::string&, int64_t, void*);
102 + static void GetPerfEvents(PerfEventCallback f, void* data) { }
103 + static void ClearPerfEvents() { }
104 + static void Init(bool enable, const std::string& events, bool subProc) {}
105 +
106 + // Normally exposed by DECLARE_THREAD_LOCAL_NO_CHECK
107 + void getCheck() { }
108 + void destroy() { }
109 + static HardwareCounter s_counter;
110 + bool m_countersSet;
111 +};
112 +
113 +#endif // NO_HARDWARE_COUNTERS
114 +
115 +///////////////////////////////////////////////////////////////////////////////
116 +}
117 +
118 +#endif
scripts/perf-counters/src/portability.h new
+187
@@ -0,0 +1,187 @@
1 +/*
2 + +----------------------------------------------------------------------+
3 + | HipHop for PHP |
4 + +----------------------------------------------------------------------+
5 + | Copyright (c) 2010-2015 Facebook, Inc. (http://www.facebook.com) |
6 + +----------------------------------------------------------------------+
7 + | This source file is subject to version 3.01 of the PHP license, |
8 + | that is bundled with this package in the file LICENSE, and is |
9 + | available through the world-wide-web at the following url: |
10 + | http://www.php.net/license/3_01.txt |
11 + | If you did not receive a copy of the PHP license and are unable to |
12 + | obtain it through the world-wide-web, please send a note to |
13 + | license@php.net so we can mail you a copy immediately. |
14 + +----------------------------------------------------------------------+
15 +*/
16 +#ifndef incl_HPHP_PORTABILITY_H_
17 +#define incl_HPHP_PORTABILITY_H_
18 +
19 +#include <folly/Likely.h> // defining LIKELY/UNLIKELY is part of this header
20 +#include <folly/Portability.h>
21 +#include <folly/CPortability.h> // defining FOLLY_DISABLE_ADDRESS_SANITIZER
22 +
23 +//////////////////////////////////////////////////////////////////////
24 +
25 +/*
26 + * Various macros to make certain things conditional on either
27 + * compiler or architecture.
28 + *
29 + * Currently we don't *really* compile on anything other than gcc or
30 + * sometimes clang, and there are some parts of the code using
31 + * __attribute__ stuff directly, but some things go through these
32 + * macros to make it maybe easier to change later.
33 + */
34 +
35 +//////////////////////////////////////////////////////////////////////
36 +
37 +// TODO: does clang define __GNUC__ ?
38 +#ifndef __GNUC__
39 +# define __attribute__(x)
40 +#endif
41 +
42 +//////////////////////////////////////////////////////////////////////
43 +
44 +#ifdef ATTRIBUTE_UNUSED
45 +# undef ATTRIBUTE_UNUSED
46 +#endif
47 +#ifdef ATTRIBUTE_NORETURN
48 +# undef ATTRIBUTE_NORETURN
49 +#endif
50 +#ifdef ATTRIBUTE_PRINTF
51 +# undef ATTRIBUTE_PRINTF
52 +#endif
53 +#ifdef ATTRIBUTE_PRINTF_STRING
54 +# undef ATTRIBUTE_PRINTF_STRING
55 +#endif
56 +
57 +#define ATTRIBUTE_PRINTF_STRING FOLLY_PRINTF_FORMAT
58 +
59 +#ifdef _MSC_VER
60 +#define ATTRIBUTE_NORETURN __declspec(noreturn)
61 +#define ATTRIBUTE_PRINTF(a1, a2)
62 +#ifndef __thread
63 +# define __thread __declspec(thread)
64 +#endif
65 +#define ATTRIBUTE_UNUSED
66 +
67 +#define ALWAYS_INLINE __forceinline
68 +#define EXTERNALLY_VISIBLE
69 +#define FLATTEN
70 +#define NEVER_INLINE __declspec(noinline)
71 +#define UNUSED
72 +#else
73 +#define ATTRIBUTE_NORETURN __attribute__((__noreturn__))
74 +#define ATTRIBUTE_PRINTF(a1, a2) \
75 + __attribute__((__format__ (__printf__, a1, a2)))
76 +#define ATTRIBUTE_UNUSED __attribute__((__unused__))
77 +
78 +#define ALWAYS_INLINE inline __attribute__((__always_inline__))
79 +#define EXTERNALLY_VISIBLE __attribute__((__externally_visible__))
80 +#define FLATTEN __attribute__((__flatten__))
81 +#define NEVER_INLINE __attribute__((__noinline__))
82 +#define UNUSED __attribute__((__unused__))
83 +#endif
84 +
85 +#ifdef DEBUG
86 +# define DEBUG_ONLY /* nop */
87 +#else
88 +# define DEBUG_ONLY UNUSED
89 +#endif
90 +
91 +/*
92 + * We need to keep some unreferenced functions from being removed by
93 + * the linker. There is no compile time mechanism for doing this, but
94 + * by putting them in the same section as some other, referenced function
95 + * in the same file, we can keep them around.
96 + *
97 + * So this macro should be used to mark at least one function that is
98 + * referenced, and other functions that are not referenced in the same
99 + * file.
100 + *
101 + * Note: this may not work properly with LTO. We'll revisit when/if we
102 + * move to it.
103 + */
104 +#ifndef __APPLE__
105 +# define KEEP_SECTION \
106 + __attribute__((__section__(".text.keep")))
107 +#else
108 +# define KEEP_SECTION \
109 + __attribute__((__section__(".text,.text.keep")))
110 +#endif
111 +
112 +#if defined(__APPLE__)
113 +// OS X has a macro "isset" defined in this header. Force the include so we can
114 +// make sure the macro gets undef'd. (I think this also applies to BSD, but we
115 +// can cross that road when we come to it.)
116 +# include <sys/param.h>
117 +# ifdef isset
118 +# undef isset
119 +# endif
120 +#endif
121 +
122 +//////////////////////////////////////////////////////////////////////
123 +
124 +#if defined(__x86_64__)
125 +
126 +# if defined(__clang__)
127 +# define DECLARE_FRAME_POINTER(fp) \
128 + ActRec* fp; \
129 + asm volatile("mov %%rbp, %0" : "=r" (fp) ::)
130 +# else
131 +# define DECLARE_FRAME_POINTER(fp) register ActRec* fp asm("rbp");
132 +# endif
133 +
134 +#elif defined(_M_X64)
135 +
136 +// TODO: FIXME! Without this implemented properly, the JIT
137 +// will fail "pretty spectacularly".
138 +# define DECLARE_FRAME_POINTER(fp) \
139 + always_assert(false); \
140 + register ActRec* fp = nullptr;
141 +
142 +#elif defined(__AARCH64EL__)
143 +
144 +# if defined(__clang__)
145 +# error Clang implementation not done for ARM
146 +# endif
147 +# define DECLARE_FRAME_POINTER(fp) register ActRec* fp asm("x29");
148 +
149 +#elif defined(__powerpc64__)
150 +
151 +# if defined(__clang__)
152 +# error Clang implementation not done for PPC64
153 +# endif
154 +# define DECLARE_FRAME_POINTER(fp) register ActRec* fp = (ActRec*) __builtin_frame_address(0);
155 +
156 +#else
157 +
158 +# error What are the stack and frame pointers called on your architecture?
159 +
160 +#endif
161 +
162 +//////////////////////////////////////////////////////////////////////
163 +
164 +// We reserve the exit status 127 to signal a failure in the
165 +// interpreter. 127 is a valid exit code on all reasonable
166 +// architectures: POSIX requires at least 8 unsigned bits and
167 +// Windows 32 signed bits.
168 +#define HPHP_EXIT_FAILURE 127
169 +
170 +//////////////////////////////////////////////////////////////////////
171 +
172 +#if FACEBOOK
173 +// Linking in libbfd is a gigantic PITA. If you want this yourself in a non-FB
174 +// build, feel free to define HAVE_LIBBFD and specify the right options to link
175 +// in libbfd.a in the extra C++ options.
176 +#define HAVE_LIBBFD 1
177 +#endif
178 +
179 +#ifndef PACKAGE
180 +// The value doesn't matter, but it must be defined before you include
181 +// bfd.h
182 +#define PACKAGE "hhvm"
183 +#endif
184 +
185 +//////////////////////////////////////////////////////////////////////
186 +
187 +#endif
scripts/perf-counters/src/thread-local.cpp new
+108
@@ -0,0 +1,108 @@
1 +/*
2 + +----------------------------------------------------------------------+
3 + | HipHop for PHP |
4 + +----------------------------------------------------------------------+
5 + | Copyright (c) 2010-2015 Facebook, Inc. (http://www.facebook.com) |
6 + +----------------------------------------------------------------------+
7 + | This source file is subject to version 3.01 of the PHP license, |
8 + | that is bundled with this package in the file LICENSE, and is |
9 + | available through the world-wide-web at the following url: |
10 + | http://www.php.net/license/3_01.txt |
11 + | If you did not receive a copy of the PHP license and are unable to |
12 + | obtain it through the world-wide-web, please send a note to |
13 + | license@php.net so we can mail you a copy immediately. |
14 + +----------------------------------------------------------------------+
15 +*/
16 +
17 +#include "hphp/util/thread-local.h"
18 +
19 +#ifdef __linux__
20 +#include <link.h>
21 +#include <asm/prctl.h>
22 +#include <sys/prctl.h>
23 +extern "C" {
24 +extern int arch_prctl(int, unsigned long*);
25 +}
26 +#endif //__linux__
27 +
28 +namespace HPHP {
29 +
30 +#ifdef USE_GCC_FAST_TLS
31 +
32 +void ThreadLocalManager::OnThreadExit(void* p) {
33 + auto list = getList(p);
34 + p = list->head;
35 + delete list;
36 + while (p != nullptr) {
37 + auto* pNode = static_cast<ThreadLocalNode<void>*>(p);
38 + if (pNode->m_on_thread_exit_fn) {
39 + pNode->m_on_thread_exit_fn(p);
40 + }
41 + p = pNode->m_next;
42 + }
43 +}
44 +
45 +void ThreadLocalManager::PushTop(void* nodePtr, size_t nodeSize) {
46 + auto& node = *static_cast<ThreadLocalNode<void>*>(nodePtr);
47 + auto key = GetManager().m_key;
48 + auto list = getList(pthread_getspecific(key));
49 + if (UNLIKELY(!list)) {
50 + ThreadLocalSetValue(key, list = new ThreadLocalList);
51 + }
52 + node.m_next = list->head;
53 + node.m_size = nodeSize;
54 + list->head = node.m_next;
55 +}
56 +
57 +ThreadLocalManager& ThreadLocalManager::GetManager() {
58 + static ThreadLocalManager m;
59 + return m;
60 +}
61 +
62 +#ifdef __APPLE__
63 +ThreadLocalManager::ThreadLocalList::ThreadLocalList() {
64 + pthread_t self = pthread_self();
65 + handler.__routine = ThreadLocalManager::OnThreadExit;
66 + handler.__arg = this;
67 + handler.__next = self->__cleanup_stack;
68 + self->__cleanup_stack = &handler;
69 +}
70 +#endif
71 +
72 +#endif
73 +
74 +#ifdef __linux__
75 +
76 +static int visit_phdr(dl_phdr_info* info, size_t, void*) {
77 + for (size_t i = 0, n = info->dlpi_phnum; i < n; ++i) {
78 + const auto& hdr = info->dlpi_phdr[i];
79 + auto addr = info->dlpi_addr + hdr.p_vaddr;
80 + if (addr < 0x100000000LL && hdr.p_type == PT_TLS) {
81 + // found the main thread-local section
82 + assert(int(hdr.p_memsz) == hdr.p_memsz); // ensure no truncation
83 + return hdr.p_memsz;
84 + }
85 + }
86 + return 0;
87 +}
88 +
89 +std::pair<void*,size_t> getCppTdata() {
90 + uintptr_t addr;
91 + if (!arch_prctl(ARCH_GET_FS, &addr)) {
92 + // fs points to the end of the threadlocal area.
93 + size_t size = dl_iterate_phdr(&visit_phdr, nullptr);
94 + return {(void*)(addr - size), size};
95 + }
96 + return {nullptr, 0};
97 +}
98 +
99 +#else
100 +
101 +// how do you find the thread local section on your system?
102 +std::pair<void*,size_t> getCppTdata() {
103 + return {nullptr, 0};
104 +}
105 +
106 +#endif //__linux__
107 +
108 +}
scripts/perf-counters/src/thread-local.h new
+765
@@ -0,0 +1,765 @@
1 +/*
2 + +----------------------------------------------------------------------+
3 + | HipHop for PHP |
4 + +----------------------------------------------------------------------+
5 + | Copyright (c) 2010-2015 Facebook, Inc. (http://www.facebook.com) |
6 + +----------------------------------------------------------------------+
7 + | This source file is subject to version 3.01 of the PHP license, |
8 + | that is bundled with this package in the file LICENSE, and is |
9 + | available through the world-wide-web at the following url: |
10 + | http://www.php.net/license/3_01.txt |
11 + | If you did not receive a copy of the PHP license and are unable to |
12 + | obtain it through the world-wide-web, please send a note to |
13 + | license@php.net so we can mail you a copy immediately. |
14 + +----------------------------------------------------------------------+
15 +*/
16 +
17 +#ifndef incl_HPHP_THREAD_LOCAL_H_
18 +#define incl_HPHP_THREAD_LOCAL_H_
19 +
20 +#include <pthread.h>
21 +#include "hphp/util/exception.h"
22 +#include <errno.h>
23 +#include <folly/String.h>
24 +#include <type_traits>
25 +
26 +namespace HPHP {
27 +
28 +// return the location of the current thread's tdata section
29 +std::pair<void*,size_t> getCppTdata();
30 +
31 +inline uintptr_t tlsBase() {
32 + uintptr_t retval;
33 +#if defined(__x86_64__)
34 + asm ("movq %%fs:0, %0" : "=r" (retval));
35 +#elif defined(__AARCH64EL__)
36 + // mrs == "move register <-- system"
37 + // tpidr_el0 == "thread process id register for exception level 0"
38 + asm ("mrs %0, tpidr_el0" : "=r" (retval));
39 +#elif defined (__powerpc64__)
40 + asm ("xor %0,%0,%0\n\t"
41 + "or %0,%0,13\n\t"
42 + : "=r" (retval));
43 +#elif defined(_M_X64)
44 + retval = (uintptr_t)_readfsbase_u64();
45 + retval = *(uintptr_t*)(retval + 88);
46 +#else
47 +# error How do you access thread-local storage on this machine?
48 +#endif
49 + return retval;
50 +}
51 +
52 +///////////////////////////////////////////////////////////////////////////////
53 +// gcc >= 4.3.0 supports the '__thread' keyword for thread locals
54 +//
55 +// Clang seems to have added this feature, or at the very least it is ignoring
56 +// __thread keyword and compiling anyway
57 +//
58 +// On OSX, gcc does emulate TLS but in a manner that invalidates assumptions
59 +// we have made about __thread and makes accessing thread-local variables in a
60 +// JIT-friendly fashion difficult (as the compiler is doing a lot of magic that
61 +// is not contractual or documented that we would need to duplicate in emitted
62 +// code) so for now we're not going to use it. One possibility if we really
63 +// want to do this is to generate functions that access variables of interest
64 +// in ThreadLocal* (all of them are NoCheck right now) and use the bytes of
65 +// gcc's compiled functions to find the values we would need to pass to
66 +// __emutls_get_address.
67 +//
68 +// icc 13.0.0 appears to support it as well but we end up with
69 +// assembler warnings of unknown importance about incorrect section
70 +// types
71 +//
72 +// __thread on cygwin and mingw uses pthreads emulation not native tls so
73 +// the emulation for thread local must be used as well
74 +//
75 +// So we use __thread on gcc, icc and clang, unless we are on OSX. On OSX, we
76 +// use our own emulation. Use the DECLARE_THREAD_LOCAL() and
77 +// IMPLEMENT_THREAD_LOCAL() macros to access either __thread or the emulation
78 +// as appropriate.
79 +
80 +#if !defined(NO_TLS) && \
81 + !defined(__CYGWIN__) && !defined(__MINGW__) && \
82 + ((__llvm__ && __clang__) || \
83 + __GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ > 3) || \
84 + __INTEL_COMPILER || defined(_MSC_VER))
85 +#define USE_GCC_FAST_TLS
86 +#endif
87 +
88 +///////////////////////////////////////////////////////////////////////////////
89 +// helper
90 +
91 +inline void ThreadLocalCheckReturn(int ret, const char *funcName) {
92 + if (ret != 0) {
93 + // This is used from global constructors so the safest thing to do is just
94 + // print to stderr and exit().
95 + fprintf(stderr, "%s returned %d: %s", funcName, ret,
96 + folly::errnoStr(ret).c_str());
97 + exit(1);
98 + }
99 +}
100 +
101 +inline void ThreadLocalCreateKey(pthread_key_t *key, void (*del)(void*)) {
102 + int ret = pthread_key_create(key, del);
103 + ThreadLocalCheckReturn(ret, "pthread_key_create");
104 +}
105 +
106 +inline void ThreadLocalSetValue(pthread_key_t key, const void* value) {
107 + int ret = pthread_setspecific(key, value);
108 + ThreadLocalCheckReturn(ret, "pthread_setspecific");
109 +}
110 +
111 +#ifdef __APPLE__
112 +typedef struct __darwin_pthread_handler_rec darwin_pthread_handler;
113 +#endif
114 +
115 +///////////////////////////////////////////////////////////////////////////////
116 +
117 +/**
118 + * A thread-local object is a "global" object within a thread. This is useful
119 + * for writing apartment-threaded code, where nothing is actually shared
120 + * between different threads (hence no locking) but those variables are not
121 + * on stack in local scope. To use it, just do something like this,
122 + *
123 + * IMPLEMENT_THREAD_LOCAL(MyClass, static_object);
124 + * static_object->data_ = ...;
125 + * static_object->doSomething();
126 + *
127 + * IMPLEMENT_THREAD_LOCAL(int, static_number);
128 + * int value = *static_number;
129 + *
130 + * So, syntax-wise it's similar to pointers. The type parameter can be a
131 + * primitive types. If it's a class, there has to be a default constructor.
132 + */
133 +
134 +///////////////////////////////////////////////////////////////////////////////
135 +#if defined(USE_GCC_FAST_TLS)
136 +
137 +/**
138 + * We keep a linked list of destructors in ThreadLocalManager to be called on
139 + * thread exit. ThreadLocalNode is a node in this list.
140 + */
141 +template <typename T>
142 +struct ThreadLocalNode {
143 + T * m_p;
144 + void (*m_on_thread_exit_fn)(void * p);
145 + void * m_next;
146 + size_t m_size;
147 +};
148 +
149 +struct ThreadLocalManager {
150 + template<class T>
151 + static void PushTop(ThreadLocalNode<T>& node) {
152 + PushTop(&node, sizeof(T));
153 + }
154 + template<class F> void scan(F& mark) const;
155 +
156 +private:
157 + static void PushTop(void* node, size_t size);
158 + struct ThreadLocalList {
159 + void* head{nullptr};
160 +#ifdef __APPLE__
161 + ThreadLocalList();
162 + darwin_pthread_handler handler;
163 +#endif
164 + };
165 + static ThreadLocalList* getList(void* p) {
166 + return static_cast<ThreadLocalList*>(p);
167 + }
168 + ThreadLocalManager() : m_key(0) {
169 +#ifdef __APPLE__
170 + ThreadLocalCreateKey(&m_key, nullptr);
171 +#else
172 + ThreadLocalCreateKey(&m_key, ThreadLocalManager::OnThreadExit);
173 +#endif
174 + };
175 + static void OnThreadExit(void *p);
176 + pthread_key_t m_key;
177 +
178 + static ThreadLocalManager& GetManager();
179 +};
180 +
181 +///////////////////////////////////////////////////////////////////////////////
182 +// ThreadLocal allocates by calling new without parameters and frees by calling
183 +// delete
184 +
185 +template<typename T>
186 +void ThreadLocalOnThreadExit(void * p) {
187 + ThreadLocalNode<T> * pNode = (ThreadLocalNode<T>*)p;
188 + delete pNode->m_p;
189 + pNode->m_p = nullptr;
190 +}
191 +
192 +/**
193 + * The USE_GCC_FAST_TLS implementation of ThreadLocal is just a lazy-initialized
194 + * pointer wrapper. In this case, we have one ThreadLocal object per thread.
195 + */
196 +template<typename T>
197 +struct ThreadLocal {
198 + T *get() const {
199 + if (m_node.m_p == nullptr) {
200 + const_cast<ThreadLocal<T>*>(this)->create();
201 + }
202 + return m_node.m_p;
203 + }
204 +
205 + NEVER_INLINE void create();
206 +
207 + bool isNull() const { return m_node.m_p == nullptr; }
208 +
209 + void destroy() {
210 + delete m_node.m_p;
211 + m_node.m_p = nullptr;
212 + }
213 +
214 + void nullOut() {
215 + m_node.m_p = nullptr;
216 + }
217 +
218 + T *operator->() const {
219 + return get();
220 + }
221 +
222 + T &operator*() const {
223 + return *get();
224 + }
225 +
226 + ThreadLocalNode<T> m_node;
227 +};
228 +
229 +template<typename T>
230 +void ThreadLocal<T>::create() {
231 + if (m_node.m_on_thread_exit_fn == nullptr) {
232 + m_node.m_on_thread_exit_fn = ThreadLocalOnThreadExit<T>;
233 + ThreadLocalManager::PushTop(m_node);
234 + }
235 + assert(m_node.m_p == nullptr);
236 + m_node.m_p = new T();
237 +}
238 +
239 +/**
240 + * ThreadLocalNoCheck is a pointer wrapper like ThreadLocal, except that it is
241 + * explicitly initialized with getCheck(), rather than being initialized when
242 + * it is first dereferenced.
243 + */
244 +template<typename T>
245 +struct ThreadLocalNoCheck {
246 + NEVER_INLINE T *getCheck() const;
247 + T* getNoCheck() const {
248 + assert(m_node.m_p);
249 + return m_node.m_p;
250 + }
251 +
252 + NEVER_INLINE void create();
253 +
254 + bool isNull() const { return m_node.m_p == nullptr; }
255 +
256 + void destroy() {
257 + delete m_node.m_p;
258 + m_node.m_p = nullptr;
259 + }
260 +
261 + T *operator->() const {
262 + return getNoCheck();
263 + }
264 +
265 + T &operator*() const {
266 + return *getNoCheck();
267 + }
268 +
269 + ThreadLocalNode<T> m_node;
270 +private:
271 + void setNull() { m_node.m_p = nullptr; }
272 +};
273 +
274 +template<typename T>
275 +void ThreadLocalNoCheck<T>::create() {
276 + if (m_node.m_on_thread_exit_fn == nullptr) {
277 + m_node.m_on_thread_exit_fn = ThreadLocalOnThreadExit<T>;
278 + ThreadLocalManager::PushTop(m_node);
279 + }
280 + assert(m_node.m_p == nullptr);
281 + m_node.m_p = new T();
282 +}
283 +template<typename T>
284 +T *ThreadLocalNoCheck<T>::getCheck() const {
285 + if (m_node.m_p == nullptr) {
286 + const_cast<ThreadLocalNoCheck<T>*>(this)->create();
287 + }
288 + return m_node.m_p;
289 +}
290 +
291 +
292 +///////////////////////////////////////////////////////////////////////////////
293 +// Singleton thread-local storage for T
294 +
295 +template<typename T>
296 +void ThreadLocalSingletonOnThreadExit(void *obj) {
297 + T::OnThreadExit((T*)obj);
298 +}
299 +
300 +// ThreadLocalSingleton has NoCheck property
301 +template <typename T>
302 +class ThreadLocalSingleton {
303 +public:
304 + ThreadLocalSingleton() { s_inited = true; }
305 +
306 + NEVER_INLINE static T *getCheck();
307 +
308 + static T* getNoCheck() {
309 + assert(s_inited);
310 + assert(s_singleton == (T*)&s_storage);
311 + return (T*)&s_storage;
312 + }
313 +
314 + static bool isNull() { return s_singleton == nullptr; }
315 +
316 + static void destroy() {
317 + assert(!s_singleton || s_singleton == (T*)&s_storage);
318 + T* p = s_singleton;
319 + if (p) {
320 + T::Delete(p);
321 + s_singleton = nullptr;
322 + }
323 + }
324 +
325 + T *operator->() const {
326 + return getNoCheck();
327 + }
328 +
329 + T &operator*() const {
330 + return *getNoCheck();
331 + }
332 +
333 +private:
334 + static __thread T *s_singleton;
335 + typedef typename std::aligned_storage<sizeof(T), sizeof(void*)>::type
336 + StorageType;
337 + static __thread StorageType s_storage;
338 + static bool s_inited; // no-fast-TLS requires construction so be consistent
339 +};
340 +
341 +template<typename T>
342 +bool ThreadLocalSingleton<T>::s_inited = false;
343 +
344 +template<typename T>
345 +T *ThreadLocalSingleton<T>::getCheck() {
346 + assert(s_inited);
347 + if (!s_singleton) {
348 + T* p = (T*) &s_storage;
349 + T::Create(p);
350 + s_singleton = p;
351 + }
352 + return s_singleton;
353 +}
354 +
355 +template<typename T> __thread T *ThreadLocalSingleton<T>::s_singleton;
356 +template<typename T> __thread typename ThreadLocalSingleton<T>::StorageType
357 + ThreadLocalSingleton<T>::s_storage;
358 +
359 +
360 +///////////////////////////////////////////////////////////////////////////////
361 +// some classes don't need new/delete at all
362 +
363 +template<typename T, bool throwOnNull = true>
364 +struct ThreadLocalProxy {
365 + T *get() const {
366 + if (m_p == nullptr && throwOnNull) {
367 + throw Exception("ThreadLocalProxy::get() called before set()");
368 + }
369 + return m_p;
370 + }
371 +
372 + void set(T* obj) {
373 + m_p = obj;
374 + }
375 +
376 + bool isNull() const { return m_p == nullptr; }
377 +
378 + void destroy() {
379 + m_p = nullptr;
380 + }
381 +
382 + T *operator->() const {
383 + return get();
384 + }
385 +
386 + T &operator*() const {
387 + return *get();
388 + }
389 +
390 + T * m_p;
391 +};
392 +
393 +/*
394 + * How to use the thread-local macros:
395 + *
396 + * Use DECLARE_THREAD_LOCAL to declare a *static* class field as thread local:
397 + * class SomeClass {
398 + * static DECLARE_THREAD_LOCAL(SomeFieldType, f);
399 + * }
400 + *
401 + * Use IMPLEMENT_THREAD_LOCAL in the cpp file to implement the field:
402 + * IMPLEMENT_THREAD_LOCAL(SomeFieldType, SomeClass::f);
403 + *
404 + * Remember: *Never* write IMPLEMENT_THREAD_LOCAL in a header file.
405 + */
406 +
407 +#define DECLARE_THREAD_LOCAL(T, f) \
408 + __thread HPHP::ThreadLocal<T> f
409 +#define IMPLEMENT_THREAD_LOCAL(T, f) \
410 + __thread HPHP::ThreadLocal<T> f
411 +
412 +#define DECLARE_THREAD_LOCAL_NO_CHECK(T, f) \
413 + __thread HPHP::ThreadLocalNoCheck<T> f
414 +#define IMPLEMENT_THREAD_LOCAL_NO_CHECK(T, f) \
415 + __thread HPHP::ThreadLocalNoCheck<T> f
416 +
417 +#define DECLARE_THREAD_LOCAL_PROXY(T, N, f) \
418 + __thread HPHP::ThreadLocalProxy<T, N> f
419 +#define IMPLEMENT_THREAD_LOCAL_PROXY(T, N, f) \
420 + __thread HPHP::ThreadLocalProxy<T, N> f
421 +
422 +#else /* USE_GCC_FAST_TLS */
423 +
424 +///////////////////////////////////////////////////////////////////////////////
425 +// ThreadLocal allocates by calling new() without parameters
426 +
427 +template<typename T>
428 +void ThreadLocalOnThreadExit(void *p) {
429 + delete (T*)p;
430 +}
431 +
432 +#ifdef __APPLE__
433 +// The __thread variables in class T will be freed when pthread calls
434 +// the destructor function on Mac. We can register a handler in
435 +// pthread_t->__cleanup_stack similar to pthread_cleanup_push(). The handler
436 +// will be called earlier so the __thread variables will still exist in the
437 +// handler when the thread exits.
438 +//
439 +// See the details at:
440 +// https://github.com/facebook/hhvm/issues/4444#issuecomment-92497582
441 +typedef struct __darwin_pthread_handler_rec darwin_pthread_handler;
442 +
443 +template<typename T>
444 +void ThreadLocalOnThreadCleanup(void *key) {
445 + void *obj = pthread_getspecific((pthread_key_t)key);
446 + if (obj) {
447 + ThreadLocalOnThreadExit<T>(obj);
448 + }
449 +}
450 +
451 +inline void ThreadLocalSetCleanupHandler(pthread_key_t cleanup_key,
452 + pthread_key_t key,
453 + void (*del)(void*)) {
454 + // Prevent from adding the handler for multiple times.
455 + darwin_pthread_handler *handler =
456 + (darwin_pthread_handler*)pthread_getspecific(cleanup_key);
457 + if (handler)
458 + return;
459 +
460 + pthread_t self = pthread_self();
461 +
462 + handler = new darwin_pthread_handler();
463 + handler->__routine = del;
464 + handler->__arg = (void*)key;
465 + handler->__next = self->__cleanup_stack;
466 + self->__cleanup_stack = handler;
467 +
468 + ThreadLocalSetValue(cleanup_key, handler);
469 +}
470 +#endif
471 +
472 +/**
473 + * This is the emulation version of ThreadLocal. In this case, the ThreadLocal
474 + * object is a true global, and the get() method returns a thread-dependent
475 + * pointer from pthread's thread-specific data management.
476 + */
477 +template<typename T>
478 +class ThreadLocal {
479 +public:
480 + /**
481 + * Constructor that has to be called from a thread-neutral place.
482 + */
483 + ThreadLocal() : m_key(0) {
484 +#ifdef __APPLE__
485 + ThreadLocalCreateKey(&m_key, nullptr);
486 + ThreadLocalCreateKey(&m_cleanup_key,
487 + ThreadLocalOnThreadExit<darwin_pthread_handler>);
488 +#else
489 + ThreadLocalCreateKey(&m_key, ThreadLocalOnThreadExit<T>);
490 +#endif
491 + }
492 +
493 + T *get() const {
494 + T *obj = (T*)pthread_getspecific(m_key);
495 + if (obj == nullptr) {
496 + obj = new T();
497 + ThreadLocalSetValue(m_key, obj);
498 +#ifdef __APPLE__
499 + ThreadLocalSetCleanupHandler(m_cleanup_key, m_key,
500 + ThreadLocalOnThreadCleanup<T>);
501 +#endif
502 + }
503 + return obj;
504 + }
505 +
506 + bool isNull() const { return pthread_getspecific(m_key) == nullptr; }
507 +
508 + void destroy() {
509 + delete (T*)pthread_getspecific(m_key);
510 + ThreadLocalSetValue(m_key, nullptr);
511 + }
512 +
513 + void nullOut() {
514 + ThreadLocalSetValue(m_key, nullptr);
515 + }
516 +
517 + /**
518 + * Access object's member or method through this operator overload.
519 + */
520 + T *operator->() const {
521 + return get();
522 + }
523 +
524 + T &operator*() const {
525 + return *get();
526 + }
527 +
528 +private:
529 + pthread_key_t m_key;
530 +
531 +#ifdef __APPLE__
532 + pthread_key_t m_cleanup_key;
533 +#endif
534 +};
535 +
536 +template<typename T>
537 +class ThreadLocalNoCheck {
538 +public:
539 + /**
540 + * Constructor that has to be called from a thread-neutral place.
541 + */
542 + ThreadLocalNoCheck() : m_key(0) {
543 +#ifdef __APPLE__
544 + ThreadLocalCreateKey(&m_key, nullptr);
545 + ThreadLocalCreateKey(&m_cleanup_key,
546 + ThreadLocalOnThreadExit<darwin_pthread_handler>);
547 +#else
548 + ThreadLocalCreateKey(&m_key, ThreadLocalOnThreadExit<T>);
549 +#endif
550 + }
551 +
552 + NEVER_INLINE T *getCheck() const;
553 +
554 + T* getNoCheck() const {
555 + T *obj = (T*)pthread_getspecific(m_key);
556 + assert(obj);
557 + return obj;
558 + }
559 +
560 + bool isNull() const { return pthread_getspecific(m_key) == nullptr; }
561 +
562 + void destroy() {
563 + delete (T*)pthread_getspecific(m_key);
564 + ThreadLocalSetValue(m_key, nullptr);
565 + }
566 +
567 + /**
568 + * Access object's member or method through this operator overload.
569 + */
570 + T *operator->() const {
571 + return getNoCheck();
572 + }
573 +
574 + T &operator*() const {
575 + return *getNoCheck();
576 + }
577 +
578 +public:
579 + void setNull() { ThreadLocalSetValue(m_key, nullptr); }
580 + pthread_key_t m_key;
581 +
582 +#ifdef __APPLE__
583 + pthread_key_t m_cleanup_key;
584 +#endif
585 +};
586 +
587 +template<typename T>
588 +T *ThreadLocalNoCheck<T>::getCheck() const {
589 + T *obj = (T*)pthread_getspecific(m_key);
590 + if (obj == nullptr) {
591 + obj = new T();
592 + ThreadLocalSetValue(m_key, obj);
593 +#ifdef __APPLE__
594 + ThreadLocalSetCleanupHandler(m_cleanup_key, m_key,
595 + ThreadLocalOnThreadCleanup<T>);
596 +#endif
597 + }
598 + return obj;
599 +}
600 +
601 +///////////////////////////////////////////////////////////////////////////////
602 +// Singleton thread-local storage for T
603 +
604 +template<typename T>
605 +void ThreadLocalSingletonOnThreadExit(void *obj) {
606 + T::OnThreadExit((T*)obj);
607 + free(obj);
608 +}
609 +
610 +#ifdef __APPLE__
611 +template<typename T>
612 +void ThreadLocalSingletonOnThreadCleanup(void *key) {
613 + void *obj = pthread_getspecific((pthread_key_t)key);
614 + if (obj) {
615 + ThreadLocalSingletonOnThreadExit<T>(obj);
616 + }
617 +}
618 +#endif
619 +
620 +// ThreadLocalSingleton has NoCheck property
621 +template<typename T>
622 +class ThreadLocalSingleton {
623 +public:
624 + ThreadLocalSingleton() { getKey(); }
625 +
626 + NEVER_INLINE static T *getCheck();
627 + static T* getNoCheck() {
628 + assert(s_inited);
629 + T *obj = (T*)pthread_getspecific(s_key);
630 + assert(obj);
631 + return obj;
632 + }
633 +
634 + static bool isNull() {
635 + return !s_inited || pthread_getspecific(s_key) == nullptr;
636 + }
637 +
638 + static void destroy() {
639 + void* p = pthread_getspecific(s_key);
640 + T::Delete((T*)p);
641 + free(p);
642 + ThreadLocalSetValue(s_key, nullptr);
643 + }
644 +
645 + T *operator->() const {
646 + return getNoCheck();
647 + }
648 +
649 + T &operator*() const {
650 + return *getNoCheck();
651 + }
652 +
653 +private:
654 + static pthread_key_t s_key;
655 + static bool s_inited; // pthread_key_t has no portable valid sentinel
656 +
657 +#ifdef __APPLE__
658 + static pthread_key_t s_cleanup_key;
659 +#endif
660 +
661 + static pthread_key_t getKey() {
662 + if (!s_inited) {
663 + s_inited = true;
664 +#ifdef __APPLE__
665 + ThreadLocalCreateKey(&s_key, nullptr);
666 + ThreadLocalCreateKey(&s_cleanup_key,
667 + ThreadLocalOnThreadExit<darwin_pthread_handler>);
668 +#else
669 + ThreadLocalCreateKey(&s_key, ThreadLocalSingletonOnThreadExit<T>);
670 +#endif
671 + }
672 + return s_key;
673 + }
674 +};
675 +
676 +template<typename T>
677 +T *ThreadLocalSingleton<T>::getCheck() {
678 + assert(s_inited);
679 + T *obj = (T*)pthread_getspecific(s_key);
680 + if (obj == nullptr) {
681 + obj = (T*)malloc(sizeof(T));
682 + T::Create(obj);
683 + ThreadLocalSetValue(s_key, obj);
684 +#ifdef __APPLE__
685 + ThreadLocalSetCleanupHandler(s_cleanup_key, s_key,
686 + ThreadLocalSingletonOnThreadCleanup<T>);
687 +#endif
688 + }
689 + return obj;
690 +}
691 +
692 +template<typename T>
693 +pthread_key_t ThreadLocalSingleton<T>::s_key;
694 +template<typename T>
695 +bool ThreadLocalSingleton<T>::s_inited = false;
696 +
697 +#ifdef __APPLE__
698 +template<typename T>
699 +pthread_key_t ThreadLocalSingleton<T>::s_cleanup_key;
700 +#endif
701 +
702 +///////////////////////////////////////////////////////////////////////////////
703 +// some classes don't need new/delete at all
704 +
705 +template<typename T, bool throwOnNull = true>
706 +class ThreadLocalProxy {
707 +public:
708 + /**
709 + * Constructor that has to be called from a thread-neutral place.
710 + */
711 + ThreadLocalProxy() : m_key(0) {
712 + ThreadLocalCreateKey(&m_key, nullptr);
713 + }
714 +
715 + T *get() const {
716 + T *obj = (T*)pthread_getspecific(m_key);
717 + if (obj == nullptr && throwOnNull) {
718 + throw Exception("ThreadLocalProxy::get() called before set()");
719 + }
720 + return obj;
721 + }
722 +
723 + void set(T* obj) {
724 + ThreadLocalSetValue(m_key, obj);
725 + }
726 +
727 + bool isNull() const { return pthread_getspecific(m_key) == nullptr; }
728 +
729 + void destroy() {
730 + ThreadLocalSetValue(m_key, nullptr);
731 + }
732 +
733 + /**
734 + * Access object's member or method through this operator overload.
735 + */
736 + T *operator->() const {
737 + return get();
738 + }
739 +
740 + T &operator*() const {
741 + return *get();
742 + }
743 +
744 +public:
745 + pthread_key_t m_key;
746 +};
747 +
748 +/**
749 + * The emulation version of the thread-local macros
750 + */
751 +#define DECLARE_THREAD_LOCAL(T, f) HPHP::ThreadLocal<T> f
752 +#define IMPLEMENT_THREAD_LOCAL(T, f) HPHP::ThreadLocal<T> f
753 +
754 +#define DECLARE_THREAD_LOCAL_NO_CHECK(T, f) HPHP::ThreadLocalNoCheck<T> f
755 +#define IMPLEMENT_THREAD_LOCAL_NO_CHECK(T, f) HPHP::ThreadLocalNoCheck<T> f
756 +
757 +#define DECLARE_THREAD_LOCAL_PROXY(T, N, f) HPHP::ThreadLocalProxy<T, N> f
758 +#define IMPLEMENT_THREAD_LOCAL_PROXY(T, N, f) HPHP::ThreadLocalProxy<T, N> f
759 +
760 +#endif /* USE_GCC_FAST_TLS */
761 +
762 +///////////////////////////////////////////////////////////////////////////////
763 +}
764 +
765 +#endif // incl_HPHP_THREAD_LOCAL_H_