X-Git-Url: https://git.liburcu.org/?a=blobdiff_plain;f=rculfhash.c;h=91958dd108b38f8177fc49432ffa1a139ddf1505;hb=34e01c64a78d067648d14c96ddd519b69abbeaeb;hp=dd49c500db4aa8fa77b859101b3e021ed6ecb96b;hpb=314558bf8e679784e0c4d5638ffbdfbc72595246;p=urcu.git diff --git a/rculfhash.c b/rculfhash.c index dd49c50..91958dd 100644 --- a/rculfhash.c +++ b/rculfhash.c @@ -242,15 +242,6 @@ struct partition_resize_work { unsigned long start, unsigned long len); }; -static -void _cds_lfht_add(struct cds_lfht *ht, - cds_lfht_match_fct match, - const void *key, - unsigned long size, - struct cds_lfht_node *node, - struct cds_lfht_iter *unique_ret, - int bucket); - /* * Algorithm to reverse bits in a word by lookup table, extended to * 64-bit words. @@ -414,7 +405,7 @@ unsigned int fls_u32(uint32_t x) } #endif -unsigned int fls_ulong(unsigned long x) +unsigned int cds_lfht_fls_ulong(unsigned long x) { #if (CAA_BITS_PER_LONG == 32) return fls_u32(x); @@ -427,7 +418,7 @@ unsigned int fls_ulong(unsigned long x) * Return the minimum order for which x <= (1UL << order). * Return -1 if x is 0. */ -int get_count_order_u32(uint32_t x) +int cds_lfht_get_count_order_u32(uint32_t x) { if (!x) return -1; @@ -439,12 +430,12 @@ int get_count_order_u32(uint32_t x) * Return the minimum order for which x <= (1UL << order). * Return -1 if x is 0. */ -int get_count_order_ulong(unsigned long x) +int cds_lfht_get_count_order_ulong(unsigned long x) { if (!x) return -1; - return fls_ulong(x - 1); + return cds_lfht_fls_ulong(x - 1); } static @@ -471,7 +462,7 @@ static void ht_init_nr_cpus_mask(void) * round up number of CPUs to next power of two, so we * can use & for modulo. */ - maxcpus = 1UL << get_count_order_ulong(maxcpus); + maxcpus = 1UL << cds_lfht_get_count_order_ulong(maxcpus); nr_cpus_mask = maxcpus - 1; } #else /* #if defined(HAVE_SYSCONF) */ @@ -549,7 +540,7 @@ void ht_count_add(struct cds_lfht *ht, unsigned long size, unsigned long hash) dbg_printf("add split count %lu\n", split_count); count = uatomic_add_return(&ht->count, 1UL << COUNT_COMMIT_ORDER); - if (likely(count & (count - 1))) + if (caa_likely(count & (count - 1))) return; /* Only if global count is power of 2 */ @@ -578,7 +569,7 @@ void ht_count_del(struct cds_lfht *ht, unsigned long size, unsigned long hash) dbg_printf("del split count %lu\n", split_count); count = uatomic_add_return(&ht->count, -(1UL << COUNT_COMMIT_ORDER)); - if (likely(count & (count - 1))) + if (caa_likely(count & (count - 1))) return; /* Only if global count is power of 2 */ @@ -614,7 +605,7 @@ void check_resize(struct cds_lfht *ht, unsigned long size, uint32_t chain_len) chain_len); if (chain_len >= CHAIN_LEN_RESIZE_THRESHOLD) cds_lfht_resize_lazy_grow(ht, size, - get_count_order_u32(chain_len - (CHAIN_LEN_TARGET - 1))); + cds_lfht_get_count_order_u32(chain_len - (CHAIN_LEN_TARGET - 1))); } static @@ -995,7 +986,7 @@ void partition_resize_helper(struct cds_lfht *ht, unsigned long i, } else { nr_threads = 1; } - partition_len = len >> get_count_order_ulong(nr_threads); + partition_len = len >> cds_lfht_get_count_order_ulong(nr_threads); work = calloc(nr_threads, sizeof(*work)); assert(work); for (thread = 0; thread < nr_threads; thread++) { @@ -1138,7 +1129,6 @@ void remove_table_partition(struct cds_lfht *ht, unsigned long i, assert(j >= size && j < (size << 1)); dbg_printf("remove entry: order %lu index %lu hash %lu\n", i, j, j); - fini_node->reverse_hash = bit_reverse_ulong(j); (void) _cds_lfht_del(ht, size, fini_node, 1); } ht->flavor->read_unlock(); @@ -1158,6 +1148,11 @@ void remove_table(struct cds_lfht *ht, unsigned long i, unsigned long len) partition_resize_helper(ht, i, len, remove_table_partition); } +/* + * fini_table() is never called for first_order == 0, which is why + * free_by_rcu_order == 0 can be used as criterion to know if free must + * be called. + */ static void fini_table(struct cds_lfht *ht, unsigned long first_order, unsigned long last_order) @@ -1225,7 +1220,7 @@ void cds_lfht_create_bucket(struct cds_lfht *ht, unsigned long size) node->next = flag_bucket(get_end()); node->reverse_hash = 0; - for (order = 1; order < get_count_order_ulong(size) + 1; order++) { + for (order = 1; order < cds_lfht_get_count_order_ulong(size) + 1; order++) { len = 1UL << (order - 1); cds_lfht_alloc_bucket_table(ht, order); @@ -1275,6 +1270,30 @@ struct cds_lfht *_cds_lfht_new(unsigned long init_size, if (!init_size || (init_size & (init_size - 1))) return NULL; + /* + * Memory management plugin default. + */ + if (!mm) { + if (CAA_BITS_PER_LONG > 32 + && max_nr_buckets + && max_nr_buckets <= (1ULL << 32)) { + /* + * For 64-bit architectures, with max number of + * buckets small enough not to use the entire + * 64-bit memory mapping space (and allowing a + * fair number of hash table instances), use the + * mmap allocator, which is faster than the + * order allocator. + */ + mm = &cds_lfht_mm_mmap; + } else { + /* + * The fallback is to use the order allocator. + */ + mm = &cds_lfht_mm_order; + } + } + /* max_nr_buckets == 0 for order based mm means infinite */ if (mm == &cds_lfht_mm_order && !max_nr_buckets) max_nr_buckets = 1UL << (MAX_TABLE_ORDER - 1); @@ -1299,7 +1318,7 @@ struct cds_lfht *_cds_lfht_new(unsigned long init_size, alloc_split_items_count(ht); /* this mutex should not nest in read-side C.S. */ pthread_mutex_init(&ht->resize_mutex, NULL); - order = get_count_order_ulong(init_size); + order = cds_lfht_get_count_order_ulong(init_size); ht->resize_target = 1UL << order; cds_lfht_create_bucket(ht, 1UL << order); ht->size = 1UL << order; @@ -1511,7 +1530,7 @@ int cds_lfht_delete_bucket(struct cds_lfht *ht) assert(is_bucket(node->next)); } - for (order = get_count_order_ulong(size); (long)order >= 0; order--) + for (order = cds_lfht_get_count_order_ulong(size); (long)order >= 0; order--) cds_lfht_free_bucket_table(ht, order); return 0; @@ -1596,8 +1615,8 @@ void _do_cds_lfht_grow(struct cds_lfht *ht, { unsigned long old_order, new_order; - old_order = get_count_order_ulong(old_size); - new_order = get_count_order_ulong(new_size); + old_order = cds_lfht_get_count_order_ulong(old_size); + new_order = cds_lfht_get_count_order_ulong(new_size); dbg_printf("resize from %lu (order %lu) to %lu (order %lu) buckets\n", old_size, old_order, new_size, new_order); assert(new_size > old_size); @@ -1612,8 +1631,8 @@ void _do_cds_lfht_shrink(struct cds_lfht *ht, unsigned long old_order, new_order; new_size = max(new_size, MIN_TABLE_SIZE); - old_order = get_count_order_ulong(old_size); - new_order = get_count_order_ulong(new_size); + old_order = cds_lfht_get_count_order_ulong(old_size); + new_order = cds_lfht_get_count_order_ulong(new_size); dbg_printf("resize from %lu (order %lu) to %lu (order %lu) buckets\n", old_size, old_order, new_size, new_order); assert(new_size < old_size);