lib/ringbuffer/backend.h

   1 #ifndef _LIB_RING_BUFFER_BACKEND_H
   2 #define _LIB_RING_BUFFER_BACKEND_H
   3
   4 /*
   5  * lib/ringbuffer/backend.h
   6  *
   7  * Ring buffer backend (API).
   8  *
   9  * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  10  *
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; only
  14  * version 2.1 of the License.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, write to the Free Software
  23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  24  *
  25  * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
  26  * the reader in flight recorder mode.
  27  */
  28
  29 #include <linux/types.h>
  30 #include <linux/sched.h>
  31 #include <linux/timer.h>
  32 #include <linux/wait.h>
  33 #include <linux/poll.h>
  34 #include <linux/list.h>
  35 #include <linux/fs.h>
  36 #include <linux/mm.h>
  37 #include <linux/uaccess.h>
  38
  39 /* Internal helpers */
  40 #include "../../wrapper/ringbuffer/backend_internal.h"
  41 #include "../../wrapper/ringbuffer/frontend_internal.h"
  42
  43 /* Ring buffer backend API */
  44
  45 /* Ring buffer backend access (read/write) */
  46
  47 extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb,
  48                                    size_t offset, void *dest, size_t len);
  49
  50 extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb,
  51                                           size_t offset, void __user *dest,
  52                                           size_t len);
  53
  54 extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb,
  55                                      size_t offset, void *dest, size_t len);
  56
  57 extern struct page **
  58 lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend *bufb, size_t offset,
  59                               void ***virt);
  60
  61 /*
  62  * Return the address where a given offset is located.
  63  * Should be used to get the current subbuffer header pointer. Given we know
  64  * it's never on a page boundary, it's safe to write directly to this address,
  65  * as long as the write is never bigger than a page size.
  66  */
  67 extern void *
  68 lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb,
  69                                size_t offset);
  70 extern void *
  71 lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb,
  72                                     size_t offset);
  73
  74 /**
  75  * lib_ring_buffer_write - write data to a buffer backend
  76  * @config : ring buffer instance configuration
  77  * @ctx: ring buffer context. (input arguments only)
  78  * @src : source pointer to copy from
  79  * @len : length of data to copy
  80  *
  81  * This function copies "len" bytes of data from a source pointer to a buffer
  82  * backend, at the current context offset. This is more or less a buffer
  83  * backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
  84  * if copy is crossing a page boundary.
  85  */
  86 static inline
  87 void lib_ring_buffer_write(const struct lib_ring_buffer_config *config,
  88                            struct lib_ring_buffer_ctx *ctx,
  89                            const void *src, size_t len)
  90 {
  91         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
  92         struct channel_backend *chanb = &ctx->chan->backend;
  93         size_t sbidx, index;
  94         size_t offset = ctx->buf_offset;
  95         ssize_t pagecpy;
  96         struct lib_ring_buffer_backend_pages *rpages;
  97         unsigned long sb_bindex, id;
  98
  99         if (unlikely(!len))
 100                 return;
 101         offset &= chanb->buf_size - 1;
 102         sbidx = offset >> chanb->subbuf_size_order;
 103         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 104         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 105         id = bufb->buf_wsb[sbidx].id;
 106         sb_bindex = subbuffer_id_get_index(config, id);
 107         rpages = bufb->array[sb_bindex];
 108         CHAN_WARN_ON(ctx->chan,
 109                      config->mode == RING_BUFFER_OVERWRITE
 110                      && subbuffer_id_is_noref(config, id));
 111         if (likely(pagecpy == len))
 112                 lib_ring_buffer_do_copy(config,
 113                                         rpages->p[index].virt
 114                                             + (offset & ~PAGE_MASK),
 115                                         src, len);
 116         else
 117                 _lib_ring_buffer_write(bufb, offset, src, len, 0);
 118         ctx->buf_offset += len;
 119 }
 120
 121 /**
 122  * lib_ring_buffer_memset - write len bytes of c to a buffer backend
 123  * @config : ring buffer instance configuration
 124  * @bufb : ring buffer backend
 125  * @offset : offset within the buffer
 126  * @c : the byte to copy
 127  * @len : number of bytes to copy
 128  *
 129  * This function writes "len" bytes of "c" to a buffer backend, at a specific
 130  * offset. This is more or less a buffer backend-specific memset() operation.
 131  * Calls the slow path (_ring_buffer_memset) if write is crossing a page
 132  * boundary.
 133  */
 134 static inline
 135 void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config,
 136                             struct lib_ring_buffer_ctx *ctx, int c, size_t len)
 137 {
 138
 139         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
 140         struct channel_backend *chanb = &ctx->chan->backend;
 141         size_t sbidx, index;
 142         size_t offset = ctx->buf_offset;
 143         ssize_t pagecpy;
 144         struct lib_ring_buffer_backend_pages *rpages;
 145         unsigned long sb_bindex, id;
 146
 147         if (unlikely(!len))
 148                 return;
 149         offset &= chanb->buf_size - 1;
 150         sbidx = offset >> chanb->subbuf_size_order;
 151         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 152         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 153         id = bufb->buf_wsb[sbidx].id;
 154         sb_bindex = subbuffer_id_get_index(config, id);
 155         rpages = bufb->array[sb_bindex];
 156         CHAN_WARN_ON(ctx->chan,
 157                      config->mode == RING_BUFFER_OVERWRITE
 158                      && subbuffer_id_is_noref(config, id));
 159         if (likely(pagecpy == len))
 160                 lib_ring_buffer_do_memset(rpages->p[index].virt
 161                                           + (offset & ~PAGE_MASK),
 162                                           c, len);
 163         else
 164                 _lib_ring_buffer_memset(bufb, offset, c, len, 0);
 165         ctx->buf_offset += len;
 166 }
 167
 168 /**
 169  * lib_ring_buffer_copy_from_user_inatomic - write userspace data to a buffer backend
 170  * @config : ring buffer instance configuration
 171  * @ctx: ring buffer context. (input arguments only)
 172  * @src : userspace source pointer to copy from
 173  * @len : length of data to copy
 174  *
 175  * This function copies "len" bytes of data from a userspace pointer to a
 176  * buffer backend, at the current context offset. This is more or less a buffer
 177  * backend-specific memcpy() operation. Calls the slow path
 178  * (_ring_buffer_write_from_user_inatomic) if copy is crossing a page boundary.
 179  * Disable the page fault handler to ensure we never try to take the mmap_sem.
 180  */
 181 static inline
 182 void lib_ring_buffer_copy_from_user_inatomic(const struct lib_ring_buffer_config *config,
 183                                     struct lib_ring_buffer_ctx *ctx,
 184                                     const void __user *src, size_t len)
 185 {
 186         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
 187         struct channel_backend *chanb = &ctx->chan->backend;
 188         size_t sbidx, index;
 189         size_t offset = ctx->buf_offset;
 190         ssize_t pagecpy;
 191         struct lib_ring_buffer_backend_pages *rpages;
 192         unsigned long sb_bindex, id;
 193         unsigned long ret;
 194         mm_segment_t old_fs = get_fs();
 195
 196         if (unlikely(!len))
 197                 return;
 198         offset &= chanb->buf_size - 1;
 199         sbidx = offset >> chanb->subbuf_size_order;
 200         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 201         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 202         id = bufb->buf_wsb[sbidx].id;
 203         sb_bindex = subbuffer_id_get_index(config, id);
 204         rpages = bufb->array[sb_bindex];
 205         CHAN_WARN_ON(ctx->chan,
 206                      config->mode == RING_BUFFER_OVERWRITE
 207                      && subbuffer_id_is_noref(config, id));
 208
 209         set_fs(KERNEL_DS);
 210         pagefault_disable();
 211         if (unlikely(!access_ok(VERIFY_READ, src, len)))
 212                 goto fill_buffer;
 213
 214         if (likely(pagecpy == len)) {
 215                 ret = lib_ring_buffer_do_copy_from_user_inatomic(
 216                         rpages->p[index].virt + (offset & ~PAGE_MASK),
 217                         src, len);
 218                 if (unlikely(ret > 0)) {
 219                         len -= (pagecpy - ret);
 220                         offset += (pagecpy - ret);
 221                         goto fill_buffer;
 222                 }
 223         } else {
 224                 _lib_ring_buffer_copy_from_user_inatomic(bufb, offset, src, len, 0);
 225         }
 226         pagefault_enable();
 227         set_fs(old_fs);
 228         ctx->buf_offset += len;
 229
 230         return;
 231
 232 fill_buffer:
 233         pagefault_enable();
 234         set_fs(old_fs);
 235         /*
 236          * In the error path we call the slow path version to avoid
 237          * the pollution of static inline code.
 238          */
 239         _lib_ring_buffer_memset(bufb, offset, 0, len, 0);
 240 }
 241
 242 /*
 243  * This accessor counts the number of unread records in a buffer.
 244  * It only provides a consistent value if no reads not writes are performed
 245  * concurrently.
 246  */
 247 static inline
 248 unsigned long lib_ring_buffer_get_records_unread(
 249                                 const struct lib_ring_buffer_config *config,
 250                                 struct lib_ring_buffer *buf)
 251 {
 252         struct lib_ring_buffer_backend *bufb = &buf->backend;
 253         struct lib_ring_buffer_backend_pages *pages;
 254         unsigned long records_unread = 0, sb_bindex, id;
 255         unsigned int i;
 256
 257         for (i = 0; i < bufb->chan->backend.num_subbuf; i++) {
 258                 id = bufb->buf_wsb[i].id;
 259                 sb_bindex = subbuffer_id_get_index(config, id);
 260                 pages = bufb->array[sb_bindex];
 261                 records_unread += v_read(config, &pages->records_unread);
 262         }
 263         if (config->mode == RING_BUFFER_OVERWRITE) {
 264                 id = bufb->buf_rsb.id;
 265                 sb_bindex = subbuffer_id_get_index(config, id);
 266                 pages = bufb->array[sb_bindex];
 267                 records_unread += v_read(config, &pages->records_unread);
 268         }
 269         return records_unread;
 270 }
 271
 272 #endif /* _LIB_RING_BUFFER_BACKEND_H */