From 1cee749c44c680a4834681bc88d95a6b6c2267b1 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Wed, 5 Jun 2013 17:43:49 -0400 Subject: [PATCH] rcuja: implement ja_node_ptr fast path Signed-off-by: Mathieu Desnoyers --- rcuja/rcuja-internal.h | 31 ++++++++++++++++++++++++++++--- rcuja/rcuja.c | 28 ++++------------------------ 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/rcuja/rcuja-internal.h b/rcuja/rcuja-internal.h index 1a3346b..202f168 100644 --- a/rcuja/rcuja-internal.h +++ b/rcuja/rcuja-internal.h @@ -140,11 +140,36 @@ void ja_node_pool_2d_bitsel(struct cds_ja_inode_flag *node, unsigned long *bits) bits[1] = ((unsigned long) node & JA_POOL_1D_MASK) >> JA_TYPE_BITS; } -__attribute__((visibility("protected"))) -unsigned long ja_node_type(struct cds_ja_inode_flag *node); +/* Hardcoded pool indexes for fast path */ +#define RCU_JA_POOL_IDX_5 5 +#define RCU_JA_POOL_IDX_6 6 +static inline +struct cds_ja_inode *ja_node_ptr(struct cds_ja_inode_flag *node) +{ + unsigned long v, type_idx; + + if (!node) + return NULL; /* RCU_JA_NULL */ + v = (unsigned long) node; + type_idx = v & JA_TYPE_MASK; + + switch (type_idx) { + case RCU_JA_POOL_IDX_5: + v &= ~(JA_POOL_1D_MASK | JA_TYPE_MASK); + break; + case RCU_JA_POOL_IDX_6: + v &= ~(JA_POOL_2D_MASK | JA_POOL_1D_MASK | JA_TYPE_MASK); + break; + default: + /* RCU_JA_LINEAR or RCU_JA_PIGEON */ + v &= JA_PTR_MASK; + break; + } + return (struct cds_ja_node *) v; +} __attribute__((visibility("protected"))) -struct cds_ja_inode *ja_node_ptr(struct cds_ja_inode_flag *node); +unsigned long ja_node_type(struct cds_ja_inode_flag *node); __attribute__((visibility("protected"))) void rcuja_free_all_children(struct cds_ja_shadow_node *shadow_node, diff --git a/rcuja/rcuja.c b/rcuja/rcuja.c index 29b7d8b..c609595 100644 --- a/rcuja/rcuja.c +++ b/rcuja/rcuja.c @@ -120,7 +120,9 @@ const struct cds_ja_type ja_types[] = { { .type_class = RCU_JA_LINEAR, .min_child = 10, .max_child = ja_type_4_max_child, .max_linear_child = ja_type_4_max_linear_child, .order = 7, }, /* Pools may fill sooner than max_child */ + /* This pool is hardcoded at index 5. See ja_node_ptr(). */ { .type_class = RCU_JA_POOL, .min_child = 20, .max_child = ja_type_5_max_child, .max_linear_child = ja_type_5_max_linear_child, .order = 8, .nr_pool_order = ja_type_5_nr_pool_order, .pool_size_order = 7, }, + /* This pool is hardcoded at index 6. See ja_node_ptr(). */ { .type_class = RCU_JA_POOL, .min_child = 45, .max_child = ja_type_6_max_child, .max_linear_child = ja_type_6_max_linear_child, .order = 9, .nr_pool_order = ja_type_6_nr_pool_order, .pool_size_order = 7, }, /* @@ -168,7 +170,9 @@ const struct cds_ja_type ja_types[] = { { .type_class = RCU_JA_LINEAR, .min_child = 10, .max_child = ja_type_4_max_child, .max_linear_child = ja_type_4_max_linear_child, .order = 8, }, /* Pools may fill sooner than max_child. */ + /* This pool is hardcoded at index 5. See ja_node_ptr(). */ { .type_class = RCU_JA_POOL, .min_child = 22, .max_child = ja_type_5_max_child, .max_linear_child = ja_type_5_max_linear_child, .order = 9, .nr_pool_order = ja_type_5_nr_pool_order, .pool_size_order = 8, }, + /* This pool is hardcoded at index 6. See ja_node_ptr(). */ { .type_class = RCU_JA_POOL, .min_child = 51, .max_child = ja_type_6_max_child, .max_linear_child = ja_type_6_max_linear_child, .order = 10, .nr_pool_order = ja_type_6_nr_pool_order, .pool_size_order = 8, }, /* @@ -261,30 +265,6 @@ unsigned long ja_node_type(struct cds_ja_inode_flag *node) return type; } -struct cds_ja_inode *ja_node_ptr(struct cds_ja_inode_flag *node) -{ - unsigned long type_index = ja_node_type(node); - const struct cds_ja_type *type; - - type = &ja_types[type_index]; - switch (type->type_class) { - case RCU_JA_LINEAR: - case RCU_JA_PIGEON: /* fall-through */ - case RCU_JA_NULL: /* fall-through */ - default: /* fall-through */ - return _ja_node_mask_ptr(node); - case RCU_JA_POOL: - switch (type->nr_pool_order) { - case 1: - return (struct cds_ja_inode *) (((unsigned long) node) & ~(JA_POOL_1D_MASK | JA_TYPE_MASK)); - case 2: - return (struct cds_ja_inode *) (((unsigned long) node) & ~(JA_POOL_2D_MASK | JA_POOL_1D_MASK | JA_TYPE_MASK)); - default: - assert(0); - } - } -} - static struct cds_ja_inode *alloc_cds_ja_node(struct cds_ja *ja, const struct cds_ja_type *ja_type) -- 2.34.1