lib/ringbuffer/backend.h

   1 #ifndef _LIB_RING_BUFFER_BACKEND_H
   2 #define _LIB_RING_BUFFER_BACKEND_H
   3
   4 /*
   5  * lib/ringbuffer/backend.h
   6  *
   7  * Ring buffer backend (API).
   8  *
   9  * Copyright (C) 2010-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
  10  *
  11  * This library is free software; you can redistribute it and/or
  12  * modify it under the terms of the GNU Lesser General Public
  13  * License as published by the Free Software Foundation; only
  14  * version 2.1 of the License.
  15  *
  16  * This library is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  19  * Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public
  22  * License along with this library; if not, write to the Free Software
  23  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  24  *
  25  * Credits to Steven Rostedt for proposing to use an extra-subbuffer owned by
  26  * the reader in flight recorder mode.
  27  */
  28
  29 #include <linux/types.h>
  30 #include <linux/sched.h>
  31 #include <linux/timer.h>
  32 #include <linux/wait.h>
  33 #include <linux/poll.h>
  34 #include <linux/list.h>
  35 #include <linux/fs.h>
  36 #include <linux/mm.h>
  37 #include <linux/uaccess.h>
  38
  39 /* Internal helpers */
  40 #include "../../wrapper/ringbuffer/backend_internal.h"
  41 #include "../../wrapper/ringbuffer/frontend_internal.h"
  42
  43 /* Ring buffer backend API */
  44
  45 /* Ring buffer backend access (read/write) */
  46
  47 extern size_t lib_ring_buffer_read(struct lib_ring_buffer_backend *bufb,
  48                                    size_t offset, void *dest, size_t len);
  49
  50 extern int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend *bufb,
  51                                           size_t offset, void __user *dest,
  52                                           size_t len);
  53
  54 extern int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend *bufb,
  55                                      size_t offset, void *dest, size_t len);
  56
  57 extern struct page **
  58 lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend *bufb, size_t offset,
  59                               void ***virt);
  60
  61 /*
  62  * Return the address where a given offset is located.
  63  * Should be used to get the current subbuffer header pointer. Given we know
  64  * it's never on a page boundary, it's safe to write directly to this address,
  65  * as long as the write is never bigger than a page size.
  66  */
  67 extern void *
  68 lib_ring_buffer_offset_address(struct lib_ring_buffer_backend *bufb,
  69                                size_t offset);
  70 extern void *
  71 lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend *bufb,
  72                                     size_t offset);
  73
  74 /**
  75  * lib_ring_buffer_write - write data to a buffer backend
  76  * @config : ring buffer instance configuration
  77  * @ctx: ring buffer context. (input arguments only)
  78  * @src : source pointer to copy from
  79  * @len : length of data to copy
  80  *
  81  * This function copies "len" bytes of data from a source pointer to a buffer
  82  * backend, at the current context offset. This is more or less a buffer
  83  * backend-specific memcpy() operation. Calls the slow path (_ring_buffer_write)
  84  * if copy is crossing a page boundary.
  85  */
  86 static inline
  87 void lib_ring_buffer_write(const struct lib_ring_buffer_config *config,
  88                            struct lib_ring_buffer_ctx *ctx,
  89                            const void *src, size_t len)
  90 {
  91         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
  92         struct channel_backend *chanb = &ctx->chan->backend;
  93         size_t sbidx, index;
  94         size_t offset = ctx->buf_offset;
  95         ssize_t pagecpy;
  96         struct lib_ring_buffer_backend_pages *rpages;
  97         unsigned long sb_bindex, id;
  98
  99         if (unlikely(!len))
 100                 return;
 101         offset &= chanb->buf_size - 1;
 102         sbidx = offset >> chanb->subbuf_size_order;
 103         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 104         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 105         id = bufb->buf_wsb[sbidx].id;
 106         sb_bindex = subbuffer_id_get_index(config, id);
 107         rpages = bufb->array[sb_bindex];
 108         CHAN_WARN_ON(ctx->chan,
 109                      config->mode == RING_BUFFER_OVERWRITE
 110                      && subbuffer_id_is_noref(config, id));
 111         if (likely(pagecpy == len))
 112                 lib_ring_buffer_do_copy(config,
 113                                         rpages->p[index].virt
 114                                             + (offset & ~PAGE_MASK),
 115                                         src, len);
 116         else
 117                 _lib_ring_buffer_write(bufb, offset, src, len, 0);
 118         ctx->buf_offset += len;
 119 }
 120
 121 /**
 122  * lib_ring_buffer_memset - write len bytes of c to a buffer backend
 123  * @config : ring buffer instance configuration
 124  * @bufb : ring buffer backend
 125  * @offset : offset within the buffer
 126  * @c : the byte to copy
 127  * @len : number of bytes to copy
 128  *
 129  * This function writes "len" bytes of "c" to a buffer backend, at a specific
 130  * offset. This is more or less a buffer backend-specific memset() operation.
 131  * Calls the slow path (_ring_buffer_memset) if write is crossing a page
 132  * boundary.
 133  */
 134 static inline
 135 void lib_ring_buffer_memset(const struct lib_ring_buffer_config *config,
 136                             struct lib_ring_buffer_ctx *ctx, int c, size_t len)
 137 {
 138
 139         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
 140         struct channel_backend *chanb = &ctx->chan->backend;
 141         size_t sbidx, index;
 142         size_t offset = ctx->buf_offset;
 143         ssize_t pagecpy;
 144         struct lib_ring_buffer_backend_pages *rpages;
 145         unsigned long sb_bindex, id;
 146
 147         if (unlikely(!len))
 148                 return;
 149         offset &= chanb->buf_size - 1;
 150         sbidx = offset >> chanb->subbuf_size_order;
 151         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 152         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 153         id = bufb->buf_wsb[sbidx].id;
 154         sb_bindex = subbuffer_id_get_index(config, id);
 155         rpages = bufb->array[sb_bindex];
 156         CHAN_WARN_ON(ctx->chan,
 157                      config->mode == RING_BUFFER_OVERWRITE
 158                      && subbuffer_id_is_noref(config, id));
 159         if (likely(pagecpy == len))
 160                 lib_ring_buffer_do_memset(rpages->p[index].virt
 161                                           + (offset & ~PAGE_MASK),
 162                                           c, len);
 163         else
 164                 _lib_ring_buffer_memset(bufb, offset, c, len, 0);
 165         ctx->buf_offset += len;
 166 }
 167
 168 /*
 169  * Copy up to @len string bytes from @src to @dest. Stop whenever a NULL
 170  * terminating character is found in @src. Returns the number of bytes
 171  * copied. Does *not* terminate @dest with NULL terminating character.
 172  */
 173 static inline
 174 size_t lib_ring_buffer_do_strcpy(const struct lib_ring_buffer_config *config,
 175                 char *dest, const char *src, size_t len)
 176 {
 177         size_t count;
 178
 179         for (count = 0; count < len; count++) {
 180                 char c;
 181
 182                 /*
 183                  * Only read source character once, in case it is
 184                  * modified concurrently.
 185                  */
 186                 c = ACCESS_ONCE(src[count]);
 187                 if (!c)
 188                         break;
 189                 lib_ring_buffer_do_copy(config, &dest[count], &c, 1);
 190         }
 191         return count;
 192 }
 193
 194 /*
 195  * Copy up to @len string bytes from @src to @dest. Stop whenever a NULL
 196  * terminating character is found in @src, or when a fault occurs.
 197  * Returns the number of bytes copied. Does *not* terminate @dest with
 198  * NULL terminating character.
 199  *
 200  * This function deals with userspace pointers, it should never be called
 201  * directly without having the src pointer checked with access_ok()
 202  * previously.
 203  */
 204 static inline
 205 size_t lib_ring_buffer_do_strcpy_from_user_inatomic(const struct lib_ring_buffer_config *config,
 206                 char *dest, const char __user *src, size_t len)
 207 {
 208         size_t count;
 209
 210         for (count = 0; count < len; count++) {
 211                 int ret;
 212                 char c;
 213
 214                 ret = __get_user(c, &src[count]);
 215                 if (ret || !c)
 216                         break;
 217                 lib_ring_buffer_do_copy(config, &dest[count], &c, 1);
 218         }
 219         return count;
 220 }
 221
 222 /**
 223  * lib_ring_buffer_strcpy - write string data to a buffer backend
 224  * @config : ring buffer instance configuration
 225  * @ctx: ring buffer context. (input arguments only)
 226  * @src : source pointer to copy from
 227  * @len : length of data to copy
 228  * @pad : character to use for padding
 229  *
 230  * This function copies @len - 1 bytes of string data from a source
 231  * pointer to a buffer backend, followed by a terminating '\0'
 232  * character, at the current context offset. This is more or less a
 233  * buffer backend-specific strncpy() operation. If a terminating '\0'
 234  * character is found in @src before @len - 1 characters are copied, pad
 235  * the buffer with @pad characters (e.g. '#'). Calls the slow path
 236  * (_ring_buffer_strcpy) if copy is crossing a page boundary.
 237  */
 238 static inline
 239 void lib_ring_buffer_strcpy(const struct lib_ring_buffer_config *config,
 240                            struct lib_ring_buffer_ctx *ctx,
 241                            const char *src, size_t len, int pad)
 242 {
 243         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
 244         struct channel_backend *chanb = &ctx->chan->backend;
 245         size_t sbidx, index, pagecpy;
 246         size_t offset = ctx->buf_offset;
 247         struct lib_ring_buffer_backend_pages *rpages;
 248         unsigned long sb_bindex, id;
 249
 250         if (unlikely(!len))
 251                 return;
 252         offset &= chanb->buf_size - 1;
 253         sbidx = offset >> chanb->subbuf_size_order;
 254         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 255         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 256         id = bufb->buf_wsb[sbidx].id;
 257         sb_bindex = subbuffer_id_get_index(config, id);
 258         rpages = bufb->array[sb_bindex];
 259         CHAN_WARN_ON(ctx->chan,
 260                      config->mode == RING_BUFFER_OVERWRITE
 261                      && subbuffer_id_is_noref(config, id));
 262         if (likely(pagecpy == len)) {
 263                 size_t count;
 264
 265                 count = lib_ring_buffer_do_strcpy(config,
 266                                         rpages->p[index].virt
 267                                             + (offset & ~PAGE_MASK),
 268                                         src, len - 1);
 269                 offset += count;
 270                 /* Padding */
 271                 if (unlikely(count < len - 1)) {
 272                         size_t pad_len = len - 1 - count;
 273
 274                         lib_ring_buffer_do_memset(rpages->p[index].virt
 275                                                 + (offset & ~PAGE_MASK),
 276                                         pad, pad_len);
 277                         offset += pad_len;
 278                 }
 279                 /* Ending '\0' */
 280                 lib_ring_buffer_do_memset(rpages->p[index].virt
 281                                         + (offset & ~PAGE_MASK),
 282                                 '\0', 1);
 283         } else {
 284                 _lib_ring_buffer_strcpy(bufb, offset, src, len, 0, pad);
 285         }
 286         ctx->buf_offset += len;
 287 }
 288
 289 /**
 290  * lib_ring_buffer_copy_from_user_inatomic - write userspace data to a buffer backend
 291  * @config : ring buffer instance configuration
 292  * @ctx: ring buffer context. (input arguments only)
 293  * @src : userspace source pointer to copy from
 294  * @len : length of data to copy
 295  *
 296  * This function copies "len" bytes of data from a userspace pointer to a
 297  * buffer backend, at the current context offset. This is more or less a buffer
 298  * backend-specific memcpy() operation. Calls the slow path
 299  * (_ring_buffer_write_from_user_inatomic) if copy is crossing a page boundary.
 300  * Disable the page fault handler to ensure we never try to take the mmap_sem.
 301  */
 302 static inline
 303 void lib_ring_buffer_copy_from_user_inatomic(const struct lib_ring_buffer_config *config,
 304                                     struct lib_ring_buffer_ctx *ctx,
 305                                     const void __user *src, size_t len)
 306 {
 307         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
 308         struct channel_backend *chanb = &ctx->chan->backend;
 309         size_t sbidx, index;
 310         size_t offset = ctx->buf_offset;
 311         ssize_t pagecpy;
 312         struct lib_ring_buffer_backend_pages *rpages;
 313         unsigned long sb_bindex, id;
 314         unsigned long ret;
 315         mm_segment_t old_fs = get_fs();
 316
 317         if (unlikely(!len))
 318                 return;
 319         offset &= chanb->buf_size - 1;
 320         sbidx = offset >> chanb->subbuf_size_order;
 321         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 322         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 323         id = bufb->buf_wsb[sbidx].id;
 324         sb_bindex = subbuffer_id_get_index(config, id);
 325         rpages = bufb->array[sb_bindex];
 326         CHAN_WARN_ON(ctx->chan,
 327                      config->mode == RING_BUFFER_OVERWRITE
 328                      && subbuffer_id_is_noref(config, id));
 329
 330         set_fs(KERNEL_DS);
 331         pagefault_disable();
 332         if (unlikely(!access_ok(VERIFY_READ, src, len)))
 333                 goto fill_buffer;
 334
 335         if (likely(pagecpy == len)) {
 336                 ret = lib_ring_buffer_do_copy_from_user_inatomic(
 337                         rpages->p[index].virt + (offset & ~PAGE_MASK),
 338                         src, len);
 339                 if (unlikely(ret > 0)) {
 340                         len -= (pagecpy - ret);
 341                         offset += (pagecpy - ret);
 342                         goto fill_buffer;
 343                 }
 344         } else {
 345                 _lib_ring_buffer_copy_from_user_inatomic(bufb, offset, src, len, 0);
 346         }
 347         pagefault_enable();
 348         set_fs(old_fs);
 349         ctx->buf_offset += len;
 350
 351         return;
 352
 353 fill_buffer:
 354         pagefault_enable();
 355         set_fs(old_fs);
 356         /*
 357          * In the error path we call the slow path version to avoid
 358          * the pollution of static inline code.
 359          */
 360         _lib_ring_buffer_memset(bufb, offset, 0, len, 0);
 361 }
 362
 363 /**
 364  * lib_ring_buffer_strcpy_from_user_inatomic - write userspace string data to a buffer backend
 365  * @config : ring buffer instance configuration
 366  * @ctx: ring buffer context (input arguments only)
 367  * @src : userspace source pointer to copy from
 368  * @len : length of data to copy
 369  * @pad : character to use for padding
 370  *
 371  * This function copies @len - 1 bytes of string data from a userspace
 372  * source pointer to a buffer backend, followed by a terminating '\0'
 373  * character, at the current context offset. This is more or less a
 374  * buffer backend-specific strncpy() operation. If a terminating '\0'
 375  * character is found in @src before @len - 1 characters are copied, pad
 376  * the buffer with @pad characters (e.g. '#'). Calls the slow path
 377  * (_ring_buffer_strcpy_from_user_inatomic) if copy is crossing a page
 378  * boundary. Disable the page fault handler to ensure we never try to
 379  * take the mmap_sem.
 380  */
 381 static inline
 382 void lib_ring_buffer_strcpy_from_user_inatomic(const struct lib_ring_buffer_config *config,
 383                 struct lib_ring_buffer_ctx *ctx,
 384                 const void __user *src, size_t len, int pad)
 385 {
 386         struct lib_ring_buffer_backend *bufb = &ctx->buf->backend;
 387         struct channel_backend *chanb = &ctx->chan->backend;
 388         size_t sbidx, index, pagecpy;
 389         size_t offset = ctx->buf_offset;
 390         struct lib_ring_buffer_backend_pages *rpages;
 391         unsigned long sb_bindex, id;
 392         mm_segment_t old_fs = get_fs();
 393
 394         if (unlikely(!len))
 395                 return;
 396         offset &= chanb->buf_size - 1;
 397         sbidx = offset >> chanb->subbuf_size_order;
 398         index = (offset & (chanb->subbuf_size - 1)) >> PAGE_SHIFT;
 399         pagecpy = min_t(size_t, len, (-offset) & ~PAGE_MASK);
 400         id = bufb->buf_wsb[sbidx].id;
 401         sb_bindex = subbuffer_id_get_index(config, id);
 402         rpages = bufb->array[sb_bindex];
 403         CHAN_WARN_ON(ctx->chan,
 404                      config->mode == RING_BUFFER_OVERWRITE
 405                      && subbuffer_id_is_noref(config, id));
 406
 407         set_fs(KERNEL_DS);
 408         pagefault_disable();
 409         if (unlikely(!access_ok(VERIFY_READ, src, len)))
 410                 goto fill_buffer;
 411
 412         if (likely(pagecpy == len)) {
 413                 size_t count;
 414
 415                 count = lib_ring_buffer_do_strcpy_from_user_inatomic(config,
 416                                         rpages->p[index].virt
 417                                             + (offset & ~PAGE_MASK),
 418                                         src, len - 1);
 419                 offset += count;
 420                 /* Padding */
 421                 if (unlikely(count < len - 1)) {
 422                         size_t pad_len = len - 1 - count;
 423
 424                         lib_ring_buffer_do_memset(rpages->p[index].virt
 425                                                 + (offset & ~PAGE_MASK),
 426                                         pad, pad_len);
 427                         offset += pad_len;
 428                 }
 429                 /* Ending '\0' */
 430                 lib_ring_buffer_do_memset(rpages->p[index].virt
 431                                         + (offset & ~PAGE_MASK),
 432                                 '\0', 1);
 433         } else {
 434                 _lib_ring_buffer_strcpy_from_user_inatomic(bufb, offset, src,
 435                                         len, 0, pad);
 436         }
 437         pagefault_enable();
 438         set_fs(old_fs);
 439         ctx->buf_offset += len;
 440
 441         return;
 442
 443 fill_buffer:
 444         pagefault_enable();
 445         set_fs(old_fs);
 446         /*
 447          * In the error path we call the slow path version to avoid
 448          * the pollution of static inline code.
 449          */
 450         _lib_ring_buffer_memset(bufb, offset, pad, len - 1, 0);
 451         offset += len - 1;
 452         _lib_ring_buffer_memset(bufb, offset, '\0', 1, 0);
 453 }
 454
 455 /*
 456  * This accessor counts the number of unread records in a buffer.
 457  * It only provides a consistent value if no reads not writes are performed
 458  * concurrently.
 459  */
 460 static inline
 461 unsigned long lib_ring_buffer_get_records_unread(
 462                                 const struct lib_ring_buffer_config *config,
 463                                 struct lib_ring_buffer *buf)
 464 {
 465         struct lib_ring_buffer_backend *bufb = &buf->backend;
 466         struct lib_ring_buffer_backend_pages *pages;
 467         unsigned long records_unread = 0, sb_bindex, id;
 468         unsigned int i;
 469
 470         for (i = 0; i < bufb->chan->backend.num_subbuf; i++) {
 471                 id = bufb->buf_wsb[i].id;
 472                 sb_bindex = subbuffer_id_get_index(config, id);
 473                 pages = bufb->array[sb_bindex];
 474                 records_unread += v_read(config, &pages->records_unread);
 475         }
 476         if (config->mode == RING_BUFFER_OVERWRITE) {
 477                 id = bufb->buf_rsb.id;
 478                 sb_bindex = subbuffer_id_get_index(config, id);
 479                 pages = bufb->array[sb_bindex];
 480                 records_unread += v_read(config, &pages->records_unread);
 481         }
 482         return records_unread;
 483 }
 484
 485 #endif /* _LIB_RING_BUFFER_BACKEND_H */