Fix: mmap: caches aliased on virtual addresses

[lttng-modules.git] / lib / ringbuffer / ring_buffer_frontend.c

diff --git a/lib/ringbuffer/ring_buffer_frontend.c b/lib/ringbuffer/ring_buffer_frontend.c
index e454ac97c168ac6ea6671c01b80ffda03c13c17c..0cd57581bb011168a351d044fe7fb739b536832c 100644 (file)
--- a/lib/ringbuffer/ring_buffer_frontend.c
+++ b/lib/ringbuffer/ring_buffer_frontend.c
@@ -54,6 +54,7 @@
 #include <linux/delay.h>
 #include <linux/module.h>
 #include <linux/percpu.h>
+#include <asm/cacheflush.h>
 
 #include <wrapper/ringbuffer/config.h>
 #include <wrapper/ringbuffer/backend.h>
@@ -1075,15 +1076,6 @@ int lib_ring_buffer_snapshot(struct lib_ring_buffer *buf,
        unsigned long consumed_cur, write_offset;
        int finalized;
 
-       /*
-        * First, ensure we perform a "final" flush onto the stream.  This will
-        * ensure we create a packet of padding if we encounter an empty
-        * packet. This ensures the time-stamps right before the snapshot is
-        * used as end of packet timestamp.
-        */
-       if (!buf->quiescent)
-               _lib_ring_buffer_switch_remote(buf, SWITCH_FLUSH);
-
 retry:
        finalized = ACCESS_ONCE(buf->finalized);
        /*
@@ -1189,6 +1181,47 @@ void lib_ring_buffer_move_consumer(struct lib_ring_buffer *buf,
 }
 EXPORT_SYMBOL_GPL(lib_ring_buffer_move_consumer);
 
+#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
+static void lib_ring_buffer_flush_read_subbuf_dcache(
+               const struct lib_ring_buffer_config *config,
+               struct channel *chan,
+               struct lib_ring_buffer *buf)
+{
+       struct lib_ring_buffer_backend_pages *pages;
+       unsigned long sb_bindex, id, i, nr_pages;
+
+       if (config->output != RING_BUFFER_MMAP)
+               return;
+
+       /*
+        * Architectures with caches aliased on virtual addresses may
+        * use different cache lines for the linear mapping vs
+        * user-space memory mapping. Given that the ring buffer is
+        * based on the kernel linear mapping, aligning it with the
+        * user-space mapping is not straightforward, and would require
+        * extra TLB entries. Therefore, simply flush the dcache for the
+        * entire sub-buffer before reading it.
+        */
+       id = buf->backend.buf_rsb.id;
+       sb_bindex = subbuffer_id_get_index(config, id);
+       pages = buf->backend.array[sb_bindex];
+       nr_pages = buf->backend.num_pages_per_subbuf;
+       for (i = 0; i < nr_pages; i++) {
+               struct lib_ring_buffer_backend_page *backend_page;
+
+               backend_page = &pages->p[i];
+               flush_dcache_page(pfn_to_page(backend_page->pfn));
+       }
+}
+#else
+static void lib_ring_buffer_flush_read_subbuf_dcache(
+               const struct lib_ring_buffer_config *config,
+               struct channel *chan,
+               struct lib_ring_buffer *buf)
+{
+}
+#endif
+
 /**
  * lib_ring_buffer_get_subbuf - get exclusive access to subbuffer for reading
  * @buf: ring buffer
@@ -1331,6 +1364,8 @@ retry:
        buf->get_subbuf_consumed = consumed;
        buf->get_subbuf = 1;
 
+       lib_ring_buffer_flush_read_subbuf_dcache(config, chan, buf);
+
        return 0;
 
 nodata:
@@ -1504,8 +1539,7 @@ void lib_ring_buffer_print_errors(struct channel *chan,
 /*
  * lib_ring_buffer_switch_old_start: Populate old subbuffer header.
  *
- * Only executed by SWITCH_FLUSH, which can be issued while tracing is active
- * or at buffer finalization (destroy).
+ * Only executed when the buffer is finalized, in SWITCH_FLUSH.
  */
 static
 void lib_ring_buffer_switch_old_start(struct lib_ring_buffer *buf,
@@ -1701,14 +1735,12 @@ int lib_ring_buffer_try_switch_slow(enum switch_mode mode,
                unsigned long sb_index, commit_count;
 
                /*
-                * We are performing a SWITCH_FLUSH. There may be concurrent
-                * writes into the buffer if e.g. invoked while performing a
-                * snapshot on an active trace.
+                * We are performing a SWITCH_FLUSH. At this stage, there are no
+                * concurrent writes into the buffer.
                 *
-                * If the client does not save any header information (sub-buffer
-                * header size == 0), don't switch empty subbuffer on finalize,
-                * because it is invalid to deliver a completely empty
-                * subbuffer.
+                * The client does not save any header information.  Don't
+                * switch empty subbuffer on finalize, because it is invalid to
+                * deliver a completely empty subbuffer.
                 */
                if (!config->cb.subbuffer_header_size())
                        return -1;
@@ -1873,12 +1905,20 @@ static void _lib_ring_buffer_switch_remote(struct lib_ring_buffer *buf,
        put_online_cpus();
 }
 
+/* Switch sub-buffer if current sub-buffer is non-empty. */
 void lib_ring_buffer_switch_remote(struct lib_ring_buffer *buf)
 {
        _lib_ring_buffer_switch_remote(buf, SWITCH_ACTIVE);
 }
 EXPORT_SYMBOL_GPL(lib_ring_buffer_switch_remote);
 
+/* Switch sub-buffer even if current sub-buffer is empty. */
+void lib_ring_buffer_switch_remote_empty(struct lib_ring_buffer *buf)
+{
+       _lib_ring_buffer_switch_remote(buf, SWITCH_FLUSH);
+}
+EXPORT_SYMBOL_GPL(lib_ring_buffer_switch_remote_empty);
+
 /*
  * Returns :
  * 0 if ok