lib/ringbuffer/backend.h

   1 #ifndef _LIB_RING_BUFFER_BACKEND_H
   2 #define _LIB_RING_BUFFER_BACKEND_H
   3
   4 /*
   5  * lib/ringbuffer/backend.h
   6  *
   7  * Ring buffer backend (API).
   8  *
   9  * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  10  *
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; only
  14  * version 2.1 of the License.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, write to the Free Software
  23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  24  *
  25  * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
  26  * the reader in flight recorder mode.
  27  */
  28
  29 #include <linux/types.h>
  30 #include <linux/sched.h>
  31 #include <linux/timer.h>
  32 #include <linux/wait.h>
  33 #include <linux/poll.h>
  34 #include <linux/list.h>
  35 #include <linux/fs.h>
  36 #include <linux/mm.h>
  37 #include <linux/uaccess.h>
  38
  39 /* Internal helpers */
  40 #include "../../wrapper/ringbuffer/backend_internal.h"
  41 #include "../../wrapper/ringbuffer/frontend_internal.h"
  42
  43 /* Ring buffer backend API */
  44
  45 /* Ring buffer backend access (read/write) */
  46
  47 extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb,
  48                                    size_t offset, void *dest, size_t len);
  49
  50 extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb,
  51                                           size_t offset, void __user *dest,
  52                                           size_t len);
  53
  54 extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb,
  55                                      size_t offset, void *dest, size_t len);
  56
  57 extern struct page **
  58 lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend *bufb, size_t offset,
  59                               void ***virt);
  60
  61 /*
  62  * Return the address where a given offset is located.
  63  * Should be used to get the current subbuffer header pointer. Given we know
  64  * it's never on a page boundary, it's safe to write directly to this address,
  65  * as long as the write is never bigger than a page size.
  66  */
  67 extern void *
  68 lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb,
  69                                size_t offset);
  70 extern void *
  71 lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb,
  72                                     size_t offset);
  73
  74 /**
  75  * lib_ring_buffer_write - write data to a buffer backend
  76  * @config : ring buffer instance configuration
  77  * @ctx: ring buffer context. (input arguments only)
  78  * @src : source pointer to copy from
  79  * @len : length of data to copy
  80  *
  81  * This function copies "len" bytes of data from a source pointer to a buffer
  82  * backend, at the current context offset. This is more or less a buffer
  83  * backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
  84  * if copy is crossing a page boundary.
  85  */
  86 static inline
  87 void lib_ring_buffer_write(const struct lib_ring_buffer_config *config,
  88                            struct lib_ring_buffer_ctx *ctx,
  89                            const void *src, size_t len)
  90 {
  91         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
  92         struct channel_backend *chanb = &ctx->chan->backend;
  93         size_t sbidx, index;
  94         size_t offset = ctx->buf_offset;
  95         ssize_t pagecpy;
  96         struct lib_ring_buffer_backend_pages *rpages;
  97         unsigned long sb_bindex, id;
  98
  99         offset &= chanb->buf_size - 1;
 100         sbidx = offset >> chanb->subbuf_size_order;
 101         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 102         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 103         id = bufb->buf_wsb[sbidx].id;
 104         sb_bindex = subbuffer_id_get_index(config, id);
 105         rpages = bufb->array[sb_bindex];
 106         CHAN_WARN_ON(ctx->chan,
 107                      config->mode == RING_BUFFER_OVERWRITE
 108                      && subbuffer_id_is_noref(config, id));
 109         if (likely(pagecpy == len))
 110                 lib_ring_buffer_do_copy(config,
 111                                         rpages->p[index].virt
 112                                             + (offset & ~PAGE_MASK),
 113                                         src, len);
 114         else
 115                 _lib_ring_buffer_write(bufb, offset, src, len, 0);
 116         ctx->buf_offset += len;
 117 }
 118
 119 /**
 120  * lib_ring_buffer_memset - write len bytes of c to a buffer backend
 121  * @config : ring buffer instance configuration
 122  * @bufb : ring buffer backend
 123  * @offset : offset within the buffer
 124  * @c : the byte to copy
 125  * @len : number of bytes to copy
 126  *
 127  * This function writes "len" bytes of "c" to a buffer backend, at a specific
 128  * offset. This is more or less a buffer backend-specific memset() operation.
 129  * Calls the slow path (_ring_buffer_memset) if write is crossing a page
 130  * boundary.
 131  */
 132 static inline
 133 void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config,
 134                             struct lib_ring_buffer_ctx *ctx, int c, size_t len)
 135 {
 136
 137         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
 138         struct channel_backend *chanb = &ctx->chan->backend;
 139         size_t sbidx, index;
 140         size_t offset = ctx->buf_offset;
 141         ssize_t pagecpy;
 142         struct lib_ring_buffer_backend_pages *rpages;
 143         unsigned long sb_bindex, id;
 144
 145         offset &= chanb->buf_size - 1;
 146         sbidx = offset >> chanb->subbuf_size_order;
 147         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 148         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 149         id = bufb->buf_wsb[sbidx].id;
 150         sb_bindex = subbuffer_id_get_index(config, id);
 151         rpages = bufb->array[sb_bindex];
 152         CHAN_WARN_ON(ctx->chan,
 153                      config->mode == RING_BUFFER_OVERWRITE
 154                      && subbuffer_id_is_noref(config, id));
 155         if (likely(pagecpy == len))
 156                 lib_ring_buffer_do_memset(rpages->p[index].virt
 157                                           + (offset & ~PAGE_MASK),
 158                                           c, len);
 159         else
 160                 _lib_ring_buffer_memset(bufb, offset, c, len, 0);
 161         ctx->buf_offset += len;
 162 }
 163
 164 /**
 165  * lib_ring_buffer_copy_from_user_inatomic - write userspace data to a buffer backend
 166  * @config : ring buffer instance configuration
 167  * @ctx: ring buffer context. (input arguments only)
 168  * @src : userspace source pointer to copy from
 169  * @len : length of data to copy
 170  *
 171  * This function copies "len" bytes of data from a userspace pointer to a
 172  * buffer backend, at the current context offset. This is more or less a buffer
 173  * backend-specific memcpy() operation. Calls the slow path
 174  * (_ring_buffer_write_from_user_inatomic) if copy is crossing a page boundary.
 175  * Disable the page fault handler to ensure we never try to take the mmap_sem.
 176  */
 177 static inline
 178 void lib_ring_buffer_copy_from_user_inatomic(const struct lib_ring_buffer_config *config,
 179                                     struct lib_ring_buffer_ctx *ctx,
 180                                     const void __user *src, size_t len)
 181 {
 182         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
 183         struct channel_backend *chanb = &ctx->chan->backend;
 184         size_t sbidx, index;
 185         size_t offset = ctx->buf_offset;
 186         ssize_t pagecpy;
 187         struct lib_ring_buffer_backend_pages *rpages;
 188         unsigned long sb_bindex, id;
 189         unsigned long ret;
 190         mm_segment_t old_fs = get_fs();
 191
 192         offset &= chanb->buf_size - 1;
 193         sbidx = offset >> chanb->subbuf_size_order;
 194         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 195         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 196         id = bufb->buf_wsb[sbidx].id;
 197         sb_bindex = subbuffer_id_get_index(config, id);
 198         rpages = bufb->array[sb_bindex];
 199         CHAN_WARN_ON(ctx->chan,
 200                      config->mode == RING_BUFFER_OVERWRITE
 201                      && subbuffer_id_is_noref(config, id));
 202
 203         set_fs(KERNEL_DS);
 204         pagefault_disable();
 205         if (unlikely(!access_ok(VERIFY_READ, src, len)))
 206                 goto fill_buffer;
 207
 208         if (likely(pagecpy == len)) {
 209                 ret = lib_ring_buffer_do_copy_from_user_inatomic(
 210                         rpages->p[index].virt + (offset & ~PAGE_MASK),
 211                         src, len);
 212                 if (unlikely(ret > 0)) {
 213                         len -= (pagecpy - ret);
 214                         offset += (pagecpy - ret);
 215                         goto fill_buffer;
 216                 }
 217         } else {
 218                 _lib_ring_buffer_copy_from_user_inatomic(bufb, offset, src, len, 0);
 219         }
 220         pagefault_enable();
 221         set_fs(old_fs);
 222         ctx->buf_offset += len;
 223
 224         return;
 225
 226 fill_buffer:
 227         pagefault_enable();
 228         set_fs(old_fs);
 229         /*
 230          * In the error path we call the slow path version to avoid
 231          * the pollution of static inline code.
 232          */
 233         _lib_ring_buffer_memset(bufb, offset, 0, len, 0);
 234 }
 235
 236 /*
 237  * This accessor counts the number of unread records in a buffer.
 238  * It only provides a consistent value if no reads not writes are performed
 239  * concurrently.
 240  */
 241 static inline
 242 unsigned long lib_ring_buffer_get_records_unread(
 243                                 const struct lib_ring_buffer_config *config,
 244                                 struct lib_ring_buffer *buf)
 245 {
 246         struct lib_ring_buffer_backend *bufb = &buf->backend;
 247         struct lib_ring_buffer_backend_pages *pages;
 248         unsigned long records_unread = 0, sb_bindex, id;
 249         unsigned int i;
 250
 251         for (i = 0; i < bufb->chan->backend.num_subbuf; i++) {
 252                 id = bufb->buf_wsb[i].id;
 253                 sb_bindex = subbuffer_id_get_index(config, id);
 254                 pages = bufb->array[sb_bindex];
 255                 records_unread += v_read(config, &pages->records_unread);
 256         }
 257         if (config->mode == RING_BUFFER_OVERWRITE) {
 258                 id = bufb->buf_rsb.id;
 259                 sb_bindex = subbuffer_id_get_index(config, id);
 260                 pages = bufb->array[sb_bindex];
 261                 records_unread += v_read(config, &pages->records_unread);
 262         }
 263         return records_unread;
 264 }
 265
 266 #endif /* _LIB_RING_BUFFER_BACKEND_H */