From 8a5c7efa50f9dce0360611b16323462c77f07321 Mon Sep 17 00:00:00 2001 From: Mathieu Desnoyers Date: Fri, 6 Oct 2023 11:48:01 -0400 Subject: [PATCH] Introduce sync vs unsync enablers Eliminate iteration over unmodified enablers when synchronizing the enablers vs event state. The intent is to turn a O(m*n) algorithm (m = number of enablers, n = number of event probes) into a O(n) when enabling many additional events when tracing is active. This change is done both for event enablers and for event notifier enablers. Signed-off-by: Mathieu Desnoyers Change-Id: Ifdadbafbf899ce5f3c5f8eb22409ac0c4af3139c --- src/common/events.h | 14 +++- src/lib/lttng-ust/lttng-events.c | 108 ++++++++++++++++++++++++++----- 2 files changed, 104 insertions(+), 18 deletions(-) diff --git a/src/common/events.h b/src/common/events.h index 76062828..0c2d3136 100644 --- a/src/common/events.h +++ b/src/common/events.h @@ -247,7 +247,12 @@ struct lttng_event_notifier_group { void *owner; int notification_fd; struct cds_list_head node; /* Event notifier group handle list */ - struct cds_list_head enablers_head; + + /* List of non-synchronized enablers */ + struct cds_list_head unsync_enablers_head; + /* List of synchronized enablers */ + struct cds_list_head sync_enablers_head; + struct cds_list_head event_notifiers_head; /* list of event_notifiers */ struct lttng_ust_event_notifier_ht event_notifiers_ht; /* hashtable of event_notifiers */ struct lttng_ust_ctx *ctx; /* contexts for filters. */ @@ -331,8 +336,11 @@ struct lttng_ust_session_private { struct cds_list_head events_head; /* list of events */ struct cds_list_head node; /* Session list */ - /* List of enablers */ - struct cds_list_head enablers_head; + /* List of non-synchronized enablers */ + struct cds_list_head unsync_enablers_head; + /* List of synchronized enablers */ + struct cds_list_head sync_enablers_head; + struct lttng_ust_event_ht events_ht; /* ht of events */ void *owner; /* object owner */ int tstate:1; /* Transient enable state */ diff --git a/src/lib/lttng-ust/lttng-events.c b/src/lib/lttng-ust/lttng-events.c index 320d459e..e618dce6 100644 --- a/src/lib/lttng-ust/lttng-events.c +++ b/src/lib/lttng-ust/lttng-events.c @@ -98,6 +98,36 @@ void lttng_ust_format_event_name(const struct lttng_ust_event_desc *desc, strcat(name, desc->event_name); } +static +void lttng_event_enabler_unsync(struct lttng_event_enabler *event_enabler) +{ + cds_list_move(&event_enabler->node, + &event_enabler->chan->parent->session->priv->unsync_enablers_head); +} + +static +void lttng_session_unsync_enablers(struct lttng_ust_session *session) +{ + cds_list_splice(&session->priv->sync_enablers_head, + &session->priv->unsync_enablers_head); + CDS_INIT_LIST_HEAD(&session->priv->sync_enablers_head); +} + +static +void lttng_event_notifier_enabler_unsync(struct lttng_event_notifier_enabler *event_notifier_enabler) +{ + cds_list_move(&event_notifier_enabler->node, + &event_notifier_enabler->group->unsync_enablers_head); +} + +static +void lttng_event_notifier_group_unsync_enablers(struct lttng_event_notifier_group *event_notifier_group) +{ + cds_list_splice(&event_notifier_group->sync_enablers_head, + &event_notifier_group->unsync_enablers_head); + CDS_INIT_LIST_HEAD(&event_notifier_group->sync_enablers_head); +} + /* * Called with ust lock held. */ @@ -167,7 +197,8 @@ struct lttng_ust_session *lttng_session_create(void) CDS_INIT_LIST_HEAD(&session->priv->chan_head); CDS_INIT_LIST_HEAD(&session->priv->events_head); CDS_INIT_LIST_HEAD(&session->priv->enums_head); - CDS_INIT_LIST_HEAD(&session->priv->enablers_head); + CDS_INIT_LIST_HEAD(&session->priv->unsync_enablers_head); + CDS_INIT_LIST_HEAD(&session->priv->sync_enablers_head); for (i = 0; i < LTTNG_UST_EVENT_HT_SIZE; i++) CDS_INIT_HLIST_HEAD(&session->priv->events_ht.table[i]); for (i = 0; i < LTTNG_UST_ENUM_HT_SIZE; i++) @@ -231,7 +262,8 @@ struct lttng_event_notifier_group *lttng_event_notifier_group_create(void) return NULL; } - CDS_INIT_LIST_HEAD(&event_notifier_group->enablers_head); + CDS_INIT_LIST_HEAD(&event_notifier_group->sync_enablers_head); + CDS_INIT_LIST_HEAD(&event_notifier_group->unsync_enablers_head); CDS_INIT_LIST_HEAD(&event_notifier_group->event_notifiers_head); for (i = 0; i < LTTNG_UST_EVENT_NOTIFIER_HT_SIZE; i++) CDS_INIT_HLIST_HEAD(&event_notifier_group->event_notifiers_ht.table[i]); @@ -315,7 +347,10 @@ void lttng_session_destroy(struct lttng_ust_session *session) lttng_ust_urcu_synchronize_rcu(); /* Wait for in-flight events to complete */ lttng_ust_tp_probe_prune_release_queue(); cds_list_for_each_entry_safe(event_enabler, event_tmpenabler, - &session->priv->enablers_head, node) + &session->priv->unsync_enablers_head, node) + lttng_event_enabler_destroy(event_enabler); + cds_list_for_each_entry_safe(event_enabler, event_tmpenabler, + &session->priv->sync_enablers_head, node) lttng_event_enabler_destroy(event_enabler); cds_list_for_each_entry_safe(event_recorder_priv, tmpevent_recorder_priv, &session->priv->events_head, node) @@ -349,7 +384,10 @@ void lttng_event_notifier_group_destroy( lttng_ust_urcu_synchronize_rcu(); cds_list_for_each_entry_safe(notifier_enabler, tmpnotifier_enabler, - &event_notifier_group->enablers_head, node) + &event_notifier_group->sync_enablers_head, node) + lttng_event_notifier_enabler_destroy(notifier_enabler); + cds_list_for_each_entry_safe(notifier_enabler, tmpnotifier_enabler, + &event_notifier_group->unsync_enablers_head, node) lttng_event_notifier_enabler_destroy(notifier_enabler); cds_list_for_each_entry_safe(event_notifier_priv, tmpevent_notifier_priv, @@ -1292,6 +1330,11 @@ int lttng_fix_pending_events(void) struct lttng_ust_session_private *session_priv; cds_list_for_each_entry(session_priv, &sessions, node) { + /* + * New probes have appeared, we need to re-sync all + * enablers. + */ + lttng_session_unsync_enablers(session_priv->pub); lttng_session_lazy_sync_event_enablers(session_priv->pub); } return 0; @@ -1302,6 +1345,11 @@ int lttng_fix_pending_event_notifiers(void) struct lttng_event_notifier_group *event_notifier_group; cds_list_for_each_entry(event_notifier_group, &event_notifier_groups, node) { + /* + * New probes have appeared, we need to re-sync all + * enablers. + */ + lttng_event_notifier_group_unsync_enablers(event_notifier_group); lttng_event_notifier_group_sync_enablers(event_notifier_group); } return 0; @@ -1418,7 +1466,7 @@ struct lttng_event_enabler *lttng_event_enabler_create( event_enabler->chan = chan; /* ctx left NULL */ event_enabler->base.enabled = 0; - cds_list_add(&event_enabler->node, &event_enabler->chan->parent->session->priv->enablers_head); + cds_list_add(&event_enabler->node, &event_enabler->chan->parent->session->priv->unsync_enablers_head); lttng_session_lazy_sync_event_enablers(event_enabler->chan->parent->session); return event_enabler; @@ -1457,7 +1505,7 @@ struct lttng_event_notifier_enabler *lttng_event_notifier_enabler_create( event_notifier_enabler->group = event_notifier_group; cds_list_add(&event_notifier_enabler->node, - &event_notifier_group->enablers_head); + &event_notifier_group->unsync_enablers_head); lttng_event_notifier_group_sync_enablers(event_notifier_group); @@ -1467,6 +1515,7 @@ struct lttng_event_notifier_enabler *lttng_event_notifier_enabler_create( int lttng_event_enabler_enable(struct lttng_event_enabler *event_enabler) { lttng_event_enabler_as_enabler(event_enabler)->enabled = 1; + lttng_event_enabler_unsync(event_enabler); lttng_session_lazy_sync_event_enablers(event_enabler->chan->parent->session); return 0; @@ -1475,6 +1524,7 @@ int lttng_event_enabler_enable(struct lttng_event_enabler *event_enabler) int lttng_event_enabler_disable(struct lttng_event_enabler *event_enabler) { lttng_event_enabler_as_enabler(event_enabler)->enabled = 0; + lttng_event_enabler_unsync(event_enabler); lttng_session_lazy_sync_event_enablers(event_enabler->chan->parent->session); return 0; @@ -1495,7 +1545,7 @@ int lttng_event_enabler_attach_filter_bytecode(struct lttng_event_enabler *event { _lttng_enabler_attach_filter_bytecode( lttng_event_enabler_as_enabler(event_enabler), bytecode); - + lttng_event_enabler_unsync(event_enabler); lttng_session_lazy_sync_event_enablers(event_enabler->chan->parent->session); return 0; } @@ -1515,7 +1565,7 @@ int lttng_event_enabler_attach_exclusion(struct lttng_event_enabler *event_enabl { _lttng_enabler_attach_exclusion( lttng_event_enabler_as_enabler(event_enabler), excluder); - + lttng_event_enabler_unsync(event_enabler); lttng_session_lazy_sync_event_enablers(event_enabler->chan->parent->session); return 0; } @@ -1524,6 +1574,7 @@ int lttng_event_notifier_enabler_enable( struct lttng_event_notifier_enabler *event_notifier_enabler) { lttng_event_notifier_enabler_as_enabler(event_notifier_enabler)->enabled = 1; + lttng_event_notifier_enabler_unsync(event_notifier_enabler); lttng_event_notifier_group_sync_enablers(event_notifier_enabler->group); return 0; @@ -1533,6 +1584,7 @@ int lttng_event_notifier_enabler_disable( struct lttng_event_notifier_enabler *event_notifier_enabler) { lttng_event_notifier_enabler_as_enabler(event_notifier_enabler)->enabled = 0; + lttng_event_notifier_enabler_unsync(event_notifier_enabler); lttng_event_notifier_group_sync_enablers(event_notifier_enabler->group); return 0; @@ -1545,7 +1597,7 @@ int lttng_event_notifier_enabler_attach_filter_bytecode( _lttng_enabler_attach_filter_bytecode( lttng_event_notifier_enabler_as_enabler(event_notifier_enabler), bytecode); - + lttng_event_notifier_enabler_unsync(event_notifier_enabler); lttng_event_notifier_group_sync_enablers(event_notifier_enabler->group); return 0; } @@ -1561,7 +1613,7 @@ int lttng_event_notifier_enabler_attach_capture_bytecode( /* Take ownership of bytecode */ *bytecode = NULL; event_notifier_enabler->num_captures++; - + lttng_event_notifier_enabler_unsync(event_notifier_enabler); lttng_event_notifier_group_sync_enablers(event_notifier_enabler->group); return 0; } @@ -1573,7 +1625,7 @@ int lttng_event_notifier_enabler_attach_exclusion( _lttng_enabler_attach_exclusion( lttng_event_notifier_enabler_as_enabler(event_notifier_enabler), excluder); - + lttng_event_notifier_enabler_unsync(event_notifier_enabler); lttng_event_notifier_group_sync_enablers(event_notifier_enabler->group); return 0; } @@ -1679,9 +1731,22 @@ void lttng_session_sync_event_enablers(struct lttng_ust_session *session) { struct lttng_event_enabler *event_enabler; struct lttng_ust_event_recorder_private *event_recorder_priv; + struct cds_list_head iter_list; + + /* + * lttng_event_enabler_ref_event_recorders can cause lazy probes + * to add items to the unsync_enablers_head list. Iterate on a + * local copy of that list until it is stable (empty). + */ + do { + CDS_INIT_LIST_HEAD(&iter_list); + cds_list_splice(&session->priv->unsync_enablers_head, &iter_list); + CDS_INIT_LIST_HEAD(&session->priv->unsync_enablers_head); + cds_list_for_each_entry(event_enabler, &iter_list, node) + lttng_event_enabler_ref_event_recorders(event_enabler); + cds_list_splice(&iter_list, &session->priv->sync_enablers_head); + } while (!cds_list_empty(&session->priv->unsync_enablers_head)); - cds_list_for_each_entry(event_enabler, &session->priv->enablers_head, node) - lttng_event_enabler_ref_event_recorders(event_enabler); /* * For each event, if at least one of its enablers is enabled, * and its channel and session transient states are enabled, we @@ -1885,9 +1950,22 @@ void lttng_event_notifier_group_sync_enablers(struct lttng_event_notifier_group { struct lttng_event_notifier_enabler *event_notifier_enabler; struct lttng_ust_event_notifier_private *event_notifier_priv; + struct cds_list_head iter_list; - cds_list_for_each_entry(event_notifier_enabler, &event_notifier_group->enablers_head, node) - lttng_event_notifier_enabler_ref_event_notifiers(event_notifier_enabler); + /* + * lttng_event_notifier_enabler_ref_event_notifiers can cause + * lazy probes to add items to the unsync_enablers_head list. + * Iterate on a local copy of that list until it is stable + * (empty). + */ + do { + CDS_INIT_LIST_HEAD(&iter_list); + cds_list_splice(&event_notifier_group->unsync_enablers_head, &iter_list); + CDS_INIT_LIST_HEAD(&event_notifier_group->unsync_enablers_head); + cds_list_for_each_entry(event_notifier_enabler, &iter_list, node) + lttng_event_notifier_enabler_ref_event_notifiers(event_notifier_enabler); + cds_list_splice(&iter_list, &event_notifier_group->sync_enablers_head); + } while (!cds_list_empty(&event_notifier_group->unsync_enablers_head)); /* * For each event_notifier, if at least one of its enablers is enabled, -- 2.34.1