X-Git-Url: http://git.liburcu.org/?a=blobdiff_plain;f=lib%2Fringbuffer%2Fring_buffer_frontend.c;h=f1801149ff48273980ba58ce10858e0ef20ffef4;hb=d4fe3450f4fec0be4ccb6cda97e7e33124b93a0e;hp=dc1ee450a4eb8dad6bf8f9ecff4332c65ae460d6;hpb=67e891fbe09d8bf262c8aefe15467446a20721db;p=lttng-modules.git diff --git a/lib/ringbuffer/ring_buffer_frontend.c b/lib/ringbuffer/ring_buffer_frontend.c index dc1ee450..f1801149 100644 --- a/lib/ringbuffer/ring_buffer_frontend.c +++ b/lib/ringbuffer/ring_buffer_frontend.c @@ -54,6 +54,7 @@ #include #include #include +#include #include #include @@ -64,6 +65,7 @@ #include #include #include +#include /* * Internal structure representing offsets to use at a sub-buffer switch. @@ -146,8 +148,8 @@ void lib_ring_buffer_free(struct lib_ring_buffer *buf) struct channel *chan = buf->backend.chan; lib_ring_buffer_print_errors(chan, buf, buf->backend.cpu); - kfree(buf->commit_hot); - kfree(buf->commit_cold); + lttng_kvfree(buf->commit_hot); + lttng_kvfree(buf->commit_cold); lib_ring_buffer_backend_free(&buf->backend); } @@ -244,7 +246,7 @@ int lib_ring_buffer_create(struct lib_ring_buffer *buf, return ret; buf->commit_hot = - kzalloc_node(ALIGN(sizeof(*buf->commit_hot) + lttng_kvzalloc_node(ALIGN(sizeof(*buf->commit_hot) * chan->backend.num_subbuf, 1 << INTERNODE_CACHE_SHIFT), GFP_KERNEL | __GFP_NOWARN, @@ -255,7 +257,7 @@ int lib_ring_buffer_create(struct lib_ring_buffer *buf, } buf->commit_cold = - kzalloc_node(ALIGN(sizeof(*buf->commit_cold) + lttng_kvzalloc_node(ALIGN(sizeof(*buf->commit_cold) * chan->backend.num_subbuf, 1 << INTERNODE_CACHE_SHIFT), GFP_KERNEL | __GFP_NOWARN, @@ -304,17 +306,17 @@ int lib_ring_buffer_create(struct lib_ring_buffer *buf, /* Error handling */ free_init: - kfree(buf->commit_cold); + lttng_kvfree(buf->commit_cold); free_commit: - kfree(buf->commit_hot); + lttng_kvfree(buf->commit_hot); free_chanbuf: lib_ring_buffer_backend_free(&buf->backend); return ret; } -static void switch_buffer_timer(unsigned long data) +static void switch_buffer_timer(LTTNG_TIMER_FUNC_ARG_TYPE t) { - struct lib_ring_buffer *buf = (struct lib_ring_buffer *)data; + struct lib_ring_buffer *buf = lttng_from_timer(buf, t, switch_timer); struct channel *chan = buf->backend.chan; const struct lib_ring_buffer_config *config = &chan->backend.config; @@ -339,22 +341,22 @@ static void lib_ring_buffer_start_switch_timer(struct lib_ring_buffer *buf) { struct channel *chan = buf->backend.chan; const struct lib_ring_buffer_config *config = &chan->backend.config; + unsigned int flags = 0; if (!chan->switch_timer_interval || buf->switch_timer_enabled) return; if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) - lttng_init_timer_pinned(&buf->switch_timer); - else - init_timer(&buf->switch_timer); + flags = LTTNG_TIMER_PINNED; - buf->switch_timer.function = switch_buffer_timer; + lttng_timer_setup(&buf->switch_timer, switch_buffer_timer, flags, buf); buf->switch_timer.expires = jiffies + chan->switch_timer_interval; - buf->switch_timer.data = (unsigned long)buf; + if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) add_timer_on(&buf->switch_timer, buf->backend.cpu); else add_timer(&buf->switch_timer); + buf->switch_timer_enabled = 1; } @@ -375,9 +377,9 @@ static void lib_ring_buffer_stop_switch_timer(struct lib_ring_buffer *buf) /* * Polling timer to check the channels for data. */ -static void read_buffer_timer(unsigned long data) +static void read_buffer_timer(LTTNG_TIMER_FUNC_ARG_TYPE t) { - struct lib_ring_buffer *buf = (struct lib_ring_buffer *)data; + struct lib_ring_buffer *buf = lttng_from_timer(buf, t, read_timer); struct channel *chan = buf->backend.chan; const struct lib_ring_buffer_config *config = &chan->backend.config; @@ -404,6 +406,7 @@ static void lib_ring_buffer_start_read_timer(struct lib_ring_buffer *buf) { struct channel *chan = buf->backend.chan; const struct lib_ring_buffer_config *config = &chan->backend.config; + unsigned int flags; if (config->wakeup != RING_BUFFER_WAKEUP_BY_TIMER || !chan->read_timer_interval @@ -411,18 +414,16 @@ static void lib_ring_buffer_start_read_timer(struct lib_ring_buffer *buf) return; if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) - lttng_init_timer_pinned(&buf->read_timer); - else - init_timer(&buf->read_timer); + flags = LTTNG_TIMER_PINNED; - buf->read_timer.function = read_buffer_timer; + lttng_timer_setup(&buf->read_timer, read_buffer_timer, flags, buf); buf->read_timer.expires = jiffies + chan->read_timer_interval; - buf->read_timer.data = (unsigned long)buf; if (config->alloc == RING_BUFFER_ALLOC_PER_CPU) add_timer_on(&buf->read_timer, buf->backend.cpu); else add_timer(&buf->read_timer); + buf->read_timer_enabled = 1; } @@ -982,7 +983,7 @@ void *channel_destroy(struct channel *chan) * Perform flush before writing to finalized. */ smp_wmb(); - ACCESS_ONCE(buf->finalized) = 1; + WRITE_ONCE(buf->finalized, 1); wake_up_interruptible(&buf->read_wait); } } else { @@ -996,10 +997,10 @@ void *channel_destroy(struct channel *chan) * Perform flush before writing to finalized. */ smp_wmb(); - ACCESS_ONCE(buf->finalized) = 1; + WRITE_ONCE(buf->finalized, 1); wake_up_interruptible(&buf->read_wait); } - ACCESS_ONCE(chan->finalized) = 1; + WRITE_ONCE(chan->finalized, 1); wake_up_interruptible(&chan->hp_wait); wake_up_interruptible(&chan->read_wait); priv = chan->backend.priv; @@ -1076,7 +1077,7 @@ int lib_ring_buffer_snapshot(struct lib_ring_buffer *buf, int finalized; retry: - finalized = ACCESS_ONCE(buf->finalized); + finalized = READ_ONCE(buf->finalized); /* * Read finalized before counters. */ @@ -1149,6 +1150,47 @@ void lib_ring_buffer_move_consumer(struct lib_ring_buffer *buf, } EXPORT_SYMBOL_GPL(lib_ring_buffer_move_consumer); +#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE +static void lib_ring_buffer_flush_read_subbuf_dcache( + const struct lib_ring_buffer_config *config, + struct channel *chan, + struct lib_ring_buffer *buf) +{ + struct lib_ring_buffer_backend_pages *pages; + unsigned long sb_bindex, id, i, nr_pages; + + if (config->output != RING_BUFFER_MMAP) + return; + + /* + * Architectures with caches aliased on virtual addresses may + * use different cache lines for the linear mapping vs + * user-space memory mapping. Given that the ring buffer is + * based on the kernel linear mapping, aligning it with the + * user-space mapping is not straightforward, and would require + * extra TLB entries. Therefore, simply flush the dcache for the + * entire sub-buffer before reading it. + */ + id = buf->backend.buf_rsb.id; + sb_bindex = subbuffer_id_get_index(config, id); + pages = buf->backend.array[sb_bindex]; + nr_pages = buf->backend.num_pages_per_subbuf; + for (i = 0; i < nr_pages; i++) { + struct lib_ring_buffer_backend_page *backend_page; + + backend_page = &pages->p[i]; + flush_dcache_page(pfn_to_page(backend_page->pfn)); + } +} +#else +static void lib_ring_buffer_flush_read_subbuf_dcache( + const struct lib_ring_buffer_config *config, + struct channel *chan, + struct lib_ring_buffer *buf) +{ +} +#endif + /** * lib_ring_buffer_get_subbuf - get exclusive access to subbuffer for reading * @buf: ring buffer @@ -1175,7 +1217,7 @@ int lib_ring_buffer_get_subbuf(struct lib_ring_buffer *buf, return -EBUSY; } retry: - finalized = ACCESS_ONCE(buf->finalized); + finalized = READ_ONCE(buf->finalized); /* * Read finalized before counters. */ @@ -1291,6 +1333,8 @@ retry: buf->get_subbuf_consumed = consumed; buf->get_subbuf = 1; + lib_ring_buffer_flush_read_subbuf_dcache(config, chan, buf); + return 0; nodata: @@ -1809,16 +1853,14 @@ static void _lib_ring_buffer_switch_remote(struct lib_ring_buffer *buf, } /* - * Taking lock on CPU hotplug to ensure two things: first, that the + * Disabling preemption ensures two things: first, that the * target cpu is not taken concurrently offline while we are within - * smp_call_function_single() (I don't trust that get_cpu() on the - * _local_ CPU actually inhibit CPU hotplug for the _remote_ CPU (to be - * confirmed)). Secondly, if it happens that the CPU is not online, our - * own call to lib_ring_buffer_switch_slow() needs to be protected from - * CPU hotplug handlers, which can also perform a remote subbuffer - * switch. + * smp_call_function_single(). Secondly, if it happens that the + * CPU is not online, our own call to lib_ring_buffer_switch_slow() + * needs to be protected from CPU hotplug handlers, which can + * also perform a remote subbuffer switch. */ - get_online_cpus(); + preempt_disable(); param.buf = buf; param.mode = mode; ret = smp_call_function_single(buf->backend.cpu, @@ -1827,7 +1869,7 @@ static void _lib_ring_buffer_switch_remote(struct lib_ring_buffer *buf, /* Remote CPU is offline, do it ourself. */ lib_ring_buffer_switch_slow(buf, mode); } - put_online_cpus(); + preempt_enable(); } /* Switch sub-buffer if current sub-buffer is non-empty. */