0232abdfe · cryptotaxi247/netdata-1

@@ -29,20 +29,23 @@

29 // use anonymous private mmap pages

30 #define ARAL_MMAP_PAGES_ABOVE (32ULL * 1024)

31

32 +#define ARAL_PAGE_INCOMING_PARTITIONS 4 // up to 32 (32-bits bitmap)

33 +

34 typedef struct aral_free {

35 size_t size;

36 struct aral_free *next;

37 } ARAL_FREE;

38

39 typedef struct aral_page {

40 + const char *filename;

41 + uint8_t *data;

42 +

43 bool marked;

44 bool started_marked;

45 bool mapped;

46 uint32_t size; // the allocation size of the page

42 - const char *filename;

43 - uint8_t *data;

44 -

47 uint32_t max_elements; // the number of elements that can fit on this page

48 + uint64_t elements_segmented; // fast path for acquiring new elements in this page

49

50 struct {

51 uint32_t used_elements; // the number of used elements on this page

@@ -61,7 +64,10 @@ typedef struct aral_page {

64 struct {

65 SPINLOCK spinlock;

66 ARAL_FREE *list;

64 - } incoming;

67 + char pad[48];

68 + } incoming[ARAL_PAGE_INCOMING_PARTITIONS];

69 +

70 + uint32_t incoming_partition_bitmap; // atomic

71

72 } ARAL_PAGE;

73

@@ -227,14 +233,14 @@ static inline void aral_page_available_unlock(ARAL *ar, ARAL_PAGE *page) {

233 spinlock_unlock(&page->available.spinlock);

234 }

235

230 -static inline void aral_page_incoming_lock(ARAL *ar, ARAL_PAGE *page) {

236 +static inline void aral_page_incoming_lock(ARAL *ar, ARAL_PAGE *page, size_t partition) {

237 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

232 - spinlock_lock(&page->incoming.spinlock);

238 + spinlock_lock(&page->incoming[partition].spinlock);

239 }

240

235 -static inline void aral_page_incoming_unlock(ARAL *ar, ARAL_PAGE *page) {

241 +static inline void aral_page_incoming_unlock(ARAL *ar, ARAL_PAGE *page, size_t partition) {

242 if(likely(!(ar->config.options & ARAL_LOCKLESS)))

237 - spinlock_unlock(&page->incoming.spinlock);

243 + spinlock_unlock(&page->incoming[partition].spinlock);

244 }

245

246 static inline bool aral_adders_trylock(ARAL *ar, bool marked) {

@@ -566,7 +572,10 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_

572 #endif

573

574 spinlock_init(&page->available.spinlock);

569 - spinlock_init(&page->incoming.spinlock);

575 +

576 + for(size_t p = 0; p < ARAL_PAGE_INCOMING_PARTITIONS ;p++)

577 + spinlock_init(&page->incoming[p].spinlock);

578 +

579 page->size = size;

580 page->max_elements = aral_elements_in_page_size(ar, page->size);

581 page->aral_lock.free_elements = page->max_elements;

@@ -582,14 +591,8 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_

591 __atomic_add_fetch(&ar->stats->structures.allocations, 1, __ATOMIC_RELAXED);

592 __atomic_add_fetch(&ar->stats->structures.allocated_bytes, structures_size, __ATOMIC_RELAXED);

593

585 - // link the free space to its page

586 - ARAL_FREE *fr = (ARAL_FREE *)page->data;

587 -

588 - fr->size = page->max_elements * ar->config.element_size;

589 - fr->next = NULL;

590 - page->available.list = fr;

591 -

592 - aral_free_validate_internal_check(ar, fr);

594 + // Initialize elements_segmented last with RELEASE

595 + __atomic_store_n(&page->elements_segmented, 0, __ATOMIC_RELEASE);

596

597 return page;

598 }

@@ -693,9 +696,9 @@ static inline ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool mar

696

697 DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(*head_ptr_free, page, aral_lock.prev, aral_lock.next);

698

696 -//#ifdef NETDATA_ARAL_INTERNAL_CHECKS

697 -// added = true;

698 -//#endif

699 + //#ifdef NETDATA_ARAL_INTERNAL_CHECKS

700 + // added = true;

701 + //#endif

702

703 aral_adders_lock(ar, marked);

704 ar->ops[idx].adders.allocating_elements -= aral_elements_in_page_size(ar, page_allocation_size);

@@ -721,12 +724,12 @@ static inline ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool mar

724

725 internal_fatal(marked && !page->marked, "ARAL: requested a marked page, but the page found is not marked");

726

724 -//#ifdef NETDATA_ARAL_INTERNAL_CHECKS

725 -// if(added) {

726 -// f2 = check_free_space___aral_lock_needed(ar, page, marked);

727 -// internal_fatal(f2.failed, "hey!");

728 -// }

729 -//#endif

727 + //#ifdef NETDATA_ARAL_INTERNAL_CHECKS

728 + // if(added) {

729 + // f2 = check_free_space___aral_lock_needed(ar, page, marked);

730 + // internal_fatal(f2.failed, "hey!");

731 + // }

732 + //#endif

733

734 internal_fatal(!page || !page->aral_lock.free_elements,

735 "ARAL: '%s' selected page does not have a free slot in it",

@@ -769,49 +772,50 @@ static inline ARAL_PAGE *aral_get_first_page_with_a_free_slot(ARAL *ar, bool mar

772 }

773

774 static void *aral_get_free_slot___no_lock_required(ARAL *ar, ARAL_PAGE *page, bool marked) {

772 - aral_page_available_lock(ar, page);

775 + // Try fast path first

776 + uint64_t slot = __atomic_fetch_add(&page->elements_segmented, 1, __ATOMIC_ACQUIRE);

777 + if (slot < page->max_elements) {

778 + // Fast path - we got a valid slot

779 + uint8_t *data = page->data + (slot * ar->config.element_size);

780

774 - if(!page->available.list) {

775 - aral_page_incoming_lock(ar, page);

776 - page->available.list = page->incoming.list;

777 - page->incoming.list = NULL;

778 - aral_page_incoming_unlock(ar, page);

779 - }

780 -

781 - ARAL_FREE *found_fr;

782 - found_fr = page->available.list;

783 -

784 - internal_fatal(!found_fr,

785 - "ARAL: '%s' incoming free list, cannot be NULL.", ar->config.name);

781 + // Set the page pointer after the element

782 + aral_set_page_pointer_after_element___do_NOT_have_aral_lock(ar, page, data, marked);

783

787 - internal_fatal(found_fr->size < ar->config.element_size,

788 - "ARAL: '%s' free element size %zu, cannot be smaller than %zu",

789 - ar->config.name, page->available.list->size, ar->config.element_size);

790 -

791 - // check if the remaining size (after we use this slot) is not enough for another element

792 - if(unlikely(found_fr->size - ar->config.element_size < ar->config.element_size)) {

793 - // we can use the entire free space entry

794 -

795 - page->available.list = found_fr->next;

784 + aral_element_given(ar, page);

785 + return data;

786 }

797 - else {

798 - // we can split the free space entry

787

800 - uint8_t *data = (uint8_t *)found_fr;

801 - ARAL_FREE *fr = (ARAL_FREE *)&data[ar->config.element_size];

802 -

803 - fr->size = found_fr->size - ar->config.element_size;

804 -

805 - // link the free slot first in the page

806 - fr->next = found_fr->next;

807 - page->available.list = fr;

788 + // Fall back to existing mechanism for reused memory

789 + aral_page_available_lock(ar, page);

790

809 - aral_free_validate_internal_check(ar, fr);

791 + if(!page->available.list) {

792 + uint32_t bitmap = __atomic_load_n(&page->incoming_partition_bitmap, __ATOMIC_RELAXED);

793 + if(!bitmap)

794 + fatal("ARAL: bitmap of incoming free elements cannot be empty at this point");

795 +

796 + size_t partition = __builtin_ffs((int)bitmap) - 1;

797 + // for(partition = 0; partition < ARAL_PAGE_INCOMING_PARTITIONS ; partition++) {

798 + // if (bitmap & (1U << partition))

799 + // break;

800 + // }

801 +

802 + if(partition >= ARAL_PAGE_INCOMING_PARTITIONS)

803 + fatal("ARAL: partition %zu must be smaller than %d", partition, ARAL_PAGE_INCOMING_PARTITIONS);

804 +

805 + aral_page_incoming_lock(ar, page, partition);

806 + page->available.list = page->incoming[partition].list;

807 + page->incoming[partition].list = NULL;

808 + __atomic_fetch_and(&page->incoming_partition_bitmap, ~(1U << partition), __ATOMIC_RELAXED);

809 + aral_page_incoming_unlock(ar, page, partition);

810 }

811

812 + ARAL_FREE *found_fr = page->available.list;

813 + internal_fatal(!found_fr, "ARAL: '%s' incoming free list, cannot be NULL.", ar->config.name);

814 + page->available.list = found_fr->next;

815 +

816 aral_page_available_unlock(ar, page);

817

814 - // put the page pointer after the element

818 + // Set the page pointer after the element

819 aral_set_page_pointer_after_element___do_NOT_have_aral_lock(ar, page, found_fr, marked);

820

821 aral_element_given(ar, page);

@@ -823,10 +827,12 @@ static inline void aral_add_free_slot___no_lock_required(ARAL *ar, ARAL_PAGE *pa

827 ARAL_FREE *fr = (ARAL_FREE *)ptr;

828 fr->size = ar->config.element_size;

829

826 - aral_page_incoming_lock(ar, page);

827 - fr->next = page->incoming.list;

828 - page->incoming.list = fr;

829 - aral_page_incoming_unlock(ar, page);

830 + size_t partition = gettid_cached() % ARAL_PAGE_INCOMING_PARTITIONS;

831 + aral_page_incoming_lock(ar, page, partition);

832 + fr->next = page->incoming[partition].list;

833 + page->incoming[partition].list = fr;

834 + __atomic_fetch_or(&page->incoming_partition_bitmap, 1U << partition, __ATOMIC_RELAXED);

835 + aral_page_incoming_unlock(ar, page, partition);

836 }

837

838 void *aral_callocz_internal(ARAL *ar, bool marked TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {

@@ -935,7 +941,7 @@ void aral_freez_internal(ARAL *ar, void *ptr TRACE_ALLOCATIONS_FUNCTION_DEFINITI

941 (size_t)page->max_elements,

942 (size_t)page->aral_lock.used_elements, (size_t)page->aral_lock.free_elements,

943 (size_t)page->aral_lock.used_elements + (size_t)page->aral_lock.free_elements

938 - );

944 + );

945

946 ARAL_PAGE **head_ptr = page->aral_lock.free_elements ? aral_pages_head_free(ar, page->marked) : aral_pages_head_full(ar, page->marked);

947 internal_fatal(!is_page_in_list(*head_ptr, page), "Page is not in this list");

@@ -1230,10 +1236,10 @@ void aral_by_size_release(ARAL *ar) {

1236 fatal("ARAL BY SIZE: double release detected");

1237

1238 aral_by_size_globals.array[size].refcount--;

1233 -// if(!aral_by_size_globals.array[size].refcount) {

1234 -// aral_destroy(aral_by_size_globals.array[size].ar);

1235 -// aral_by_size_globals.array[size].ar = NULL;

1236 -// }

1239 + // if(!aral_by_size_globals.array[size].refcount) {

1240 + // aral_destroy(aral_by_size_globals.array[size].ar);

1241 + // aral_by_size_globals.array[size].ar = NULL;

1242 + // }

1243

1244 spinlock_unlock(&aral_by_size_globals.spinlock);

1245 }

ARAL: fast path to quickly allocate elements on a new page (#19376)