2 * ring_buffer_backend.c
4 * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; only
9 * version 2.1 of the License.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 #include <linux/stddef.h>
22 #include <linux/module.h>
23 #include <linux/string.h>
24 #include <linux/bitops.h>
25 #include <linux/delay.h>
26 #include <linux/errno.h>
27 #include <linux/slab.h>
28 #include <linux/cpu.h>
30 #include <linux/vmalloc.h>
32 #include "../../wrapper/vmalloc.h" /* for wrapper_vmalloc_sync_all() */
33 #include "../../wrapper/ringbuffer/config.h"
34 #include "../../wrapper/ringbuffer/backend.h"
35 #include "../../wrapper/ringbuffer/frontend.h"
38 * lib_ring_buffer_backend_allocate - allocate a channel buffer
39 * @config: ring buffer instance configuration
40 * @buf: the buffer struct
41 * @size: total size of the buffer
42 * @num_subbuf: number of subbuffers
43 * @extra_reader_sb: need extra subbuffer for reader
46 int lib_ring_buffer_backend_allocate(const struct lib_ring_buffer_config
*config
,
47 struct lib_ring_buffer_backend
*bufb
,
48 size_t size
, size_t num_subbuf
,
51 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
52 unsigned long j
, num_pages
, num_pages_per_subbuf
, page_idx
= 0;
53 unsigned long subbuf_size
, mmap_offset
= 0;
54 unsigned long num_subbuf_alloc
;
59 num_pages
= size
>> PAGE_SHIFT
;
60 num_pages_per_subbuf
= num_pages
>> get_count_order(num_subbuf
);
61 subbuf_size
= chanb
->subbuf_size
;
62 num_subbuf_alloc
= num_subbuf
;
64 if (extra_reader_sb
) {
65 num_pages
+= num_pages_per_subbuf
; /* Add pages for reader */
69 pages
= vmalloc_node(ALIGN(sizeof(*pages
) * num_pages
,
70 1 << INTERNODE_CACHE_SHIFT
),
71 cpu_to_node(max(bufb
->cpu
, 0)));
75 virt
= vmalloc_node(ALIGN(sizeof(*virt
) * num_pages
,
76 1 << INTERNODE_CACHE_SHIFT
),
77 cpu_to_node(max(bufb
->cpu
, 0)));
81 bufb
->array
= kmalloc_node(ALIGN(sizeof(*bufb
->array
)
83 1 << INTERNODE_CACHE_SHIFT
),
84 GFP_KERNEL
| __GFP_NOWARN
,
85 cpu_to_node(max(bufb
->cpu
, 0)));
86 if (unlikely(!bufb
->array
))
89 for (i
= 0; i
< num_pages
; i
++) {
90 pages
[i
] = alloc_pages_node(cpu_to_node(max(bufb
->cpu
, 0)),
91 GFP_KERNEL
| __GFP_NOWARN
| __GFP_ZERO
, 0);
92 if (unlikely(!pages
[i
]))
94 virt
[i
] = page_address(pages
[i
]);
96 bufb
->num_pages_per_subbuf
= num_pages_per_subbuf
;
98 /* Allocate backend pages array elements */
99 for (i
= 0; i
< num_subbuf_alloc
; i
++) {
102 sizeof(struct lib_ring_buffer_backend_pages
) +
103 sizeof(struct lib_ring_buffer_backend_page
)
104 * num_pages_per_subbuf
,
105 1 << INTERNODE_CACHE_SHIFT
),
106 GFP_KERNEL
| __GFP_NOWARN
,
107 cpu_to_node(max(bufb
->cpu
, 0)));
112 /* Allocate write-side subbuffer table */
113 bufb
->buf_wsb
= kzalloc_node(ALIGN(
114 sizeof(struct lib_ring_buffer_backend_subbuffer
)
116 1 << INTERNODE_CACHE_SHIFT
),
117 GFP_KERNEL
| __GFP_NOWARN
,
118 cpu_to_node(max(bufb
->cpu
, 0)));
119 if (unlikely(!bufb
->buf_wsb
))
122 for (i
= 0; i
< num_subbuf
; i
++)
123 bufb
->buf_wsb
[i
].id
= subbuffer_id(config
, 0, 1, i
);
125 /* Assign read-side subbuffer table */
127 bufb
->buf_rsb
.id
= subbuffer_id(config
, 0, 1,
128 num_subbuf_alloc
- 1);
130 bufb
->buf_rsb
.id
= subbuffer_id(config
, 0, 1, 0);
132 /* Assign pages to page index */
133 for (i
= 0; i
< num_subbuf_alloc
; i
++) {
134 for (j
= 0; j
< num_pages_per_subbuf
; j
++) {
135 CHAN_WARN_ON(chanb
, page_idx
> num_pages
);
136 bufb
->array
[i
]->p
[j
].virt
= virt
[page_idx
];
137 bufb
->array
[i
]->p
[j
].page
= pages
[page_idx
];
140 if (config
->output
== RING_BUFFER_MMAP
) {
141 bufb
->array
[i
]->mmap_offset
= mmap_offset
;
142 mmap_offset
+= subbuf_size
;
147 * If kmalloc ever uses vmalloc underneath, make sure the buffer pages
150 wrapper_vmalloc_sync_all();
156 for (i
= 0; (i
< num_subbuf_alloc
&& bufb
->array
[i
]); i
++)
157 kfree(bufb
->array
[i
]);
159 /* Free all allocated pages */
160 for (i
= 0; (i
< num_pages
&& pages
[i
]); i
++)
161 __free_page(pages
[i
]);
171 int lib_ring_buffer_backend_create(struct lib_ring_buffer_backend
*bufb
,
172 struct channel_backend
*chanb
, int cpu
)
174 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
176 bufb
->chan
= container_of(chanb
, struct channel
, backend
);
179 return lib_ring_buffer_backend_allocate(config
, bufb
, chanb
->buf_size
,
181 chanb
->extra_reader_sb
);
184 void lib_ring_buffer_backend_free(struct lib_ring_buffer_backend
*bufb
)
186 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
187 unsigned long i
, j
, num_subbuf_alloc
;
189 num_subbuf_alloc
= chanb
->num_subbuf
;
190 if (chanb
->extra_reader_sb
)
193 kfree(bufb
->buf_wsb
);
194 for (i
= 0; i
< num_subbuf_alloc
; i
++) {
195 for (j
= 0; j
< bufb
->num_pages_per_subbuf
; j
++)
196 __free_page(bufb
->array
[i
]->p
[j
].page
);
197 kfree(bufb
->array
[i
]);
203 void lib_ring_buffer_backend_reset(struct lib_ring_buffer_backend
*bufb
)
205 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
206 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
207 unsigned long num_subbuf_alloc
;
210 num_subbuf_alloc
= chanb
->num_subbuf
;
211 if (chanb
->extra_reader_sb
)
214 for (i
= 0; i
< chanb
->num_subbuf
; i
++)
215 bufb
->buf_wsb
[i
].id
= subbuffer_id(config
, 0, 1, i
);
216 if (chanb
->extra_reader_sb
)
217 bufb
->buf_rsb
.id
= subbuffer_id(config
, 0, 1,
218 num_subbuf_alloc
- 1);
220 bufb
->buf_rsb
.id
= subbuffer_id(config
, 0, 1, 0);
222 for (i
= 0; i
< num_subbuf_alloc
; i
++) {
223 /* Don't reset mmap_offset */
224 v_set(config
, &bufb
->array
[i
]->records_commit
, 0);
225 v_set(config
, &bufb
->array
[i
]->records_unread
, 0);
226 bufb
->array
[i
]->data_size
= 0;
227 /* Don't reset backend page and virt addresses */
229 /* Don't reset num_pages_per_subbuf, cpu, allocated */
230 v_set(config
, &bufb
->records_read
, 0);
234 * The frontend is responsible for also calling ring_buffer_backend_reset for
235 * each buffer when calling channel_backend_reset.
237 void channel_backend_reset(struct channel_backend
*chanb
)
239 struct channel
*chan
= container_of(chanb
, struct channel
, backend
);
240 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
243 * Don't reset buf_size, subbuf_size, subbuf_size_order,
244 * num_subbuf_order, buf_size_order, extra_reader_sb, num_subbuf,
245 * priv, notifiers, config, cpumask and name.
247 chanb
->start_tsc
= config
->cb
.ring_buffer_clock_read(chan
);
250 #ifdef CONFIG_HOTPLUG_CPU
252 * lib_ring_buffer_cpu_hp_callback - CPU hotplug callback
253 * @nb: notifier block
254 * @action: hotplug action to take
257 * Returns the success/failure of the operation. (%NOTIFY_OK, %NOTIFY_BAD)
260 int lib_ring_buffer_cpu_hp_callback(struct notifier_block
*nb
,
261 unsigned long action
,
264 unsigned int cpu
= (unsigned long)hcpu
;
265 struct channel_backend
*chanb
= container_of(nb
, struct channel_backend
,
267 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
268 struct lib_ring_buffer
*buf
;
271 CHAN_WARN_ON(chanb
, config
->alloc
== RING_BUFFER_ALLOC_GLOBAL
);
275 case CPU_UP_PREPARE_FROZEN
:
276 buf
= per_cpu_ptr(chanb
->buf
, cpu
);
277 ret
= lib_ring_buffer_create(buf
, chanb
, cpu
);
280 "ring_buffer_cpu_hp_callback: cpu %d "
281 "buffer creation failed\n", cpu
);
286 case CPU_DEAD_FROZEN
:
287 /* No need to do a buffer switch here, because it will happen
288 * when tracing is stopped, or will be done by switch timer CPU
297 * channel_backend_init - initialize a channel backend
298 * @chanb: channel backend
299 * @name: channel name
300 * @config: client ring buffer configuration
301 * @priv: client private data
302 * @parent: dentry of parent directory, %NULL for root directory
303 * @subbuf_size: size of sub-buffers (> PAGE_SIZE, power of 2)
304 * @num_subbuf: number of sub-buffers (power of 2)
306 * Returns channel pointer if successful, %NULL otherwise.
308 * Creates per-cpu channel buffers using the sizes and attributes
309 * specified. The created channel buffer files will be named
310 * name_0...name_N-1. File permissions will be %S_IRUSR.
312 * Called with CPU hotplug disabled.
314 int channel_backend_init(struct channel_backend
*chanb
,
316 const struct lib_ring_buffer_config
*config
,
317 void *priv
, size_t subbuf_size
, size_t num_subbuf
)
319 struct channel
*chan
= container_of(chanb
, struct channel
, backend
);
326 /* Check that the subbuffer size is larger than a page. */
327 if (subbuf_size
< PAGE_SIZE
)
331 * Make sure the number of subbuffers and subbuffer size are
332 * power of 2 and nonzero.
334 if (!subbuf_size
|| (subbuf_size
& (subbuf_size
- 1)))
336 if (!num_subbuf
|| (num_subbuf
& (num_subbuf
- 1)))
339 * Overwrite mode buffers require at least 2 subbuffers per
342 if (config
->mode
== RING_BUFFER_OVERWRITE
&& num_subbuf
< 2)
345 ret
= subbuffer_id_check_index(config
, num_subbuf
);
350 chanb
->buf_size
= num_subbuf
* subbuf_size
;
351 chanb
->subbuf_size
= subbuf_size
;
352 chanb
->buf_size_order
= get_count_order(chanb
->buf_size
);
353 chanb
->subbuf_size_order
= get_count_order(subbuf_size
);
354 chanb
->num_subbuf_order
= get_count_order(num_subbuf
);
355 chanb
->extra_reader_sb
=
356 (config
->mode
== RING_BUFFER_OVERWRITE
) ? 1 : 0;
357 chanb
->num_subbuf
= num_subbuf
;
358 strlcpy(chanb
->name
, name
, NAME_MAX
);
359 memcpy(&chanb
->config
, config
, sizeof(chanb
->config
));
361 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
) {
362 if (!zalloc_cpumask_var(&chanb
->cpumask
, GFP_KERNEL
))
366 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
) {
367 /* Allocating the buffer per-cpu structures */
368 chanb
->buf
= alloc_percpu(struct lib_ring_buffer
);
373 * In case of non-hotplug cpu, if the ring-buffer is allocated
374 * in early initcall, it will not be notified of secondary cpus.
375 * In that off case, we need to allocate for all possible cpus.
377 #ifdef CONFIG_HOTPLUG_CPU
379 * buf->backend.allocated test takes care of concurrent CPU
381 * Priority higher than frontend, so we create the ring buffer
382 * before we start the timer.
384 chanb
->cpu_hp_notifier
.notifier_call
=
385 lib_ring_buffer_cpu_hp_callback
;
386 chanb
->cpu_hp_notifier
.priority
= 5;
387 register_hotcpu_notifier(&chanb
->cpu_hp_notifier
);
390 for_each_online_cpu(i
) {
391 ret
= lib_ring_buffer_create(per_cpu_ptr(chanb
->buf
, i
),
394 goto free_bufs
; /* cpu hotplug locked */
398 for_each_possible_cpu(i
) {
399 ret
= lib_ring_buffer_create(per_cpu_ptr(chanb
->buf
, i
),
402 goto free_bufs
; /* cpu hotplug locked */
406 chanb
->buf
= kzalloc(sizeof(struct lib_ring_buffer
), GFP_KERNEL
);
409 ret
= lib_ring_buffer_create(chanb
->buf
, chanb
, -1);
413 chanb
->start_tsc
= config
->cb
.ring_buffer_clock_read(chan
);
418 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
) {
419 for_each_possible_cpu(i
) {
420 struct lib_ring_buffer
*buf
= per_cpu_ptr(chanb
->buf
, i
);
422 if (!buf
->backend
.allocated
)
424 lib_ring_buffer_free(buf
);
426 #ifdef CONFIG_HOTPLUG_CPU
428 unregister_hotcpu_notifier(&chanb
->cpu_hp_notifier
);
430 free_percpu(chanb
->buf
);
434 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
)
435 free_cpumask_var(chanb
->cpumask
);
440 * channel_backend_unregister_notifiers - unregister notifiers
445 void channel_backend_unregister_notifiers(struct channel_backend
*chanb
)
447 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
449 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
)
450 unregister_hotcpu_notifier(&chanb
->cpu_hp_notifier
);
454 * channel_backend_free - destroy the channel
457 * Destroy all channel buffers and frees the channel.
459 void channel_backend_free(struct channel_backend
*chanb
)
461 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
464 if (config
->alloc
== RING_BUFFER_ALLOC_PER_CPU
) {
465 for_each_possible_cpu(i
) {
466 struct lib_ring_buffer
*buf
= per_cpu_ptr(chanb
->buf
, i
);
468 if (!buf
->backend
.allocated
)
470 lib_ring_buffer_free(buf
);
472 free_cpumask_var(chanb
->cpumask
);
473 free_percpu(chanb
->buf
);
475 struct lib_ring_buffer
*buf
= chanb
->buf
;
477 CHAN_WARN_ON(chanb
, !buf
->backend
.allocated
);
478 lib_ring_buffer_free(buf
);
484 * lib_ring_buffer_write - write data to a ring_buffer buffer.
485 * @bufb : buffer backend
486 * @offset : offset within the buffer
487 * @src : source address
488 * @len : length to write
489 * @pagecpy : page size copied so far
491 void _lib_ring_buffer_write(struct lib_ring_buffer_backend
*bufb
, size_t offset
,
492 const void *src
, size_t len
, size_t pagecpy
)
494 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
495 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
497 struct lib_ring_buffer_backend_pages
*rpages
;
498 unsigned long sb_bindex
, id
;
504 sbidx
= offset
>> chanb
->subbuf_size_order
;
505 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
508 * Underlying layer should never ask for writes across
511 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
513 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
514 id
= bufb
->buf_wsb
[sbidx
].id
;
515 sb_bindex
= subbuffer_id_get_index(config
, id
);
516 rpages
= bufb
->array
[sb_bindex
];
517 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
518 && subbuffer_id_is_noref(config
, id
));
519 lib_ring_buffer_do_copy(config
,
520 rpages
->p
[index
].virt
521 + (offset
& ~PAGE_MASK
),
523 } while (unlikely(len
!= pagecpy
));
525 EXPORT_SYMBOL_GPL(_lib_ring_buffer_write
);
529 * lib_ring_buffer_memset - write len bytes of c to a ring_buffer buffer.
530 * @bufb : buffer backend
531 * @offset : offset within the buffer
532 * @c : the byte to write
533 * @len : length to write
534 * @pagecpy : page size copied so far
536 void _lib_ring_buffer_memset(struct lib_ring_buffer_backend
*bufb
,
538 int c
, size_t len
, size_t pagecpy
)
540 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
541 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
543 struct lib_ring_buffer_backend_pages
*rpages
;
544 unsigned long sb_bindex
, id
;
549 sbidx
= offset
>> chanb
->subbuf_size_order
;
550 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
553 * Underlying layer should never ask for writes across
556 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
558 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
559 id
= bufb
->buf_wsb
[sbidx
].id
;
560 sb_bindex
= subbuffer_id_get_index(config
, id
);
561 rpages
= bufb
->array
[sb_bindex
];
562 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
563 && subbuffer_id_is_noref(config
, id
));
564 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
565 + (offset
& ~PAGE_MASK
),
567 } while (unlikely(len
!= pagecpy
));
569 EXPORT_SYMBOL_GPL(_lib_ring_buffer_memset
);
572 * lib_ring_buffer_strcpy - write string data to a ring_buffer buffer.
573 * @bufb : buffer backend
574 * @offset : offset within the buffer
575 * @src : source address
576 * @len : length to write
577 * @pagecpy : page size copied so far
578 * @pad : character to use for padding
580 void _lib_ring_buffer_strcpy(struct lib_ring_buffer_backend
*bufb
,
581 size_t offset
, const char *src
, size_t len
,
582 size_t pagecpy
, int pad
)
584 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
585 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
587 struct lib_ring_buffer_backend_pages
*rpages
;
588 unsigned long sb_bindex
, id
;
589 int src_terminated
= 0;
591 CHAN_WARN_ON(chanb
, !len
);
597 sbidx
= offset
>> chanb
->subbuf_size_order
;
598 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
601 * Underlying layer should never ask for writes across
604 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
606 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
607 id
= bufb
->buf_wsb
[sbidx
].id
;
608 sb_bindex
= subbuffer_id_get_index(config
, id
);
609 rpages
= bufb
->array
[sb_bindex
];
610 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
611 && subbuffer_id_is_noref(config
, id
));
613 if (likely(!src_terminated
)) {
614 size_t count
, to_copy
;
618 to_copy
--; /* Final '\0' */
619 count
= lib_ring_buffer_do_strcpy(config
,
620 rpages
->p
[index
].virt
621 + (offset
& ~PAGE_MASK
),
625 if (unlikely(count
< to_copy
)) {
626 size_t pad_len
= to_copy
- count
;
628 /* Next pages will have padding */
630 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
631 + (offset
& ~PAGE_MASK
),
640 pad_len
--; /* Final '\0' */
641 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
642 + (offset
& ~PAGE_MASK
),
646 } while (unlikely(len
!= pagecpy
));
648 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
),
651 EXPORT_SYMBOL_GPL(_lib_ring_buffer_strcpy
);
654 * lib_ring_buffer_copy_from_user_inatomic - write user data to a ring_buffer buffer.
655 * @bufb : buffer backend
656 * @offset : offset within the buffer
657 * @src : source address
658 * @len : length to write
659 * @pagecpy : page size copied so far
661 * This function deals with userspace pointers, it should never be called
662 * directly without having the src pointer checked with access_ok()
665 void _lib_ring_buffer_copy_from_user_inatomic(struct lib_ring_buffer_backend
*bufb
,
667 const void __user
*src
, size_t len
,
670 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
671 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
673 struct lib_ring_buffer_backend_pages
*rpages
;
674 unsigned long sb_bindex
, id
;
681 sbidx
= offset
>> chanb
->subbuf_size_order
;
682 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
685 * Underlying layer should never ask for writes across
688 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
690 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
691 id
= bufb
->buf_wsb
[sbidx
].id
;
692 sb_bindex
= subbuffer_id_get_index(config
, id
);
693 rpages
= bufb
->array
[sb_bindex
];
694 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
695 && subbuffer_id_is_noref(config
, id
));
696 ret
= lib_ring_buffer_do_copy_from_user_inatomic(rpages
->p
[index
].virt
697 + (offset
& ~PAGE_MASK
),
701 _lib_ring_buffer_memset(bufb
, offset
, 0, len
, 0);
702 break; /* stop copy */
704 } while (unlikely(len
!= pagecpy
));
706 EXPORT_SYMBOL_GPL(_lib_ring_buffer_copy_from_user_inatomic
);
709 * lib_ring_buffer_strcpy_from_user_inatomic - write userspace string data to a ring_buffer buffer.
710 * @bufb : buffer backend
711 * @offset : offset within the buffer
712 * @src : source address
713 * @len : length to write
714 * @pagecpy : page size copied so far
715 * @pad : character to use for padding
717 * This function deals with userspace pointers, it should never be called
718 * directly without having the src pointer checked with access_ok()
721 void _lib_ring_buffer_strcpy_from_user_inatomic(struct lib_ring_buffer_backend
*bufb
,
722 size_t offset
, const char __user
*src
, size_t len
,
723 size_t pagecpy
, int pad
)
725 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
726 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
728 struct lib_ring_buffer_backend_pages
*rpages
;
729 unsigned long sb_bindex
, id
;
730 int src_terminated
= 0;
737 sbidx
= offset
>> chanb
->subbuf_size_order
;
738 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
741 * Underlying layer should never ask for writes across
744 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
746 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
747 id
= bufb
->buf_wsb
[sbidx
].id
;
748 sb_bindex
= subbuffer_id_get_index(config
, id
);
749 rpages
= bufb
->array
[sb_bindex
];
750 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
751 && subbuffer_id_is_noref(config
, id
));
753 if (likely(!src_terminated
)) {
754 size_t count
, to_copy
;
758 to_copy
--; /* Final '\0' */
759 count
= lib_ring_buffer_do_strcpy_from_user_inatomic(config
,
760 rpages
->p
[index
].virt
761 + (offset
& ~PAGE_MASK
),
765 if (unlikely(count
< to_copy
)) {
766 size_t pad_len
= to_copy
- count
;
768 /* Next pages will have padding */
770 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
771 + (offset
& ~PAGE_MASK
),
780 pad_len
--; /* Final '\0' */
781 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
782 + (offset
& ~PAGE_MASK
),
786 } while (unlikely(len
!= pagecpy
));
788 lib_ring_buffer_do_memset(rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
),
791 EXPORT_SYMBOL_GPL(_lib_ring_buffer_strcpy_from_user_inatomic
);
794 * lib_ring_buffer_read - read data from ring_buffer_buffer.
795 * @bufb : buffer backend
796 * @offset : offset within the buffer
797 * @dest : destination address
798 * @len : length to copy to destination
800 * Should be protected by get_subbuf/put_subbuf.
801 * Returns the length copied.
803 size_t lib_ring_buffer_read(struct lib_ring_buffer_backend
*bufb
, size_t offset
,
804 void *dest
, size_t len
)
806 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
807 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
808 size_t index
, pagecpy
, orig_len
;
809 struct lib_ring_buffer_backend_pages
*rpages
;
810 unsigned long sb_bindex
, id
;
813 offset
&= chanb
->buf_size
- 1;
814 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
818 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
819 id
= bufb
->buf_rsb
.id
;
820 sb_bindex
= subbuffer_id_get_index(config
, id
);
821 rpages
= bufb
->array
[sb_bindex
];
822 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
823 && subbuffer_id_is_noref(config
, id
));
824 memcpy(dest
, rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
),
831 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
833 * Underlying layer should never ask for reads across
836 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
840 EXPORT_SYMBOL_GPL(lib_ring_buffer_read
);
843 * __lib_ring_buffer_copy_to_user - read data from ring_buffer to userspace
844 * @bufb : buffer backend
845 * @offset : offset within the buffer
846 * @dest : destination userspace address
847 * @len : length to copy to destination
849 * Should be protected by get_subbuf/put_subbuf.
850 * access_ok() must have been performed on dest addresses prior to call this
852 * Returns -EFAULT on error, 0 if ok.
854 int __lib_ring_buffer_copy_to_user(struct lib_ring_buffer_backend
*bufb
,
855 size_t offset
, void __user
*dest
, size_t len
)
857 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
858 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
861 struct lib_ring_buffer_backend_pages
*rpages
;
862 unsigned long sb_bindex
, id
;
864 offset
&= chanb
->buf_size
- 1;
865 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
869 pagecpy
= min_t(size_t, len
, PAGE_SIZE
- (offset
& ~PAGE_MASK
));
870 id
= bufb
->buf_rsb
.id
;
871 sb_bindex
= subbuffer_id_get_index(config
, id
);
872 rpages
= bufb
->array
[sb_bindex
];
873 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
874 && subbuffer_id_is_noref(config
, id
));
875 if (__copy_to_user(dest
,
876 rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
),
884 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
886 * Underlying layer should never ask for reads across
889 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
893 EXPORT_SYMBOL_GPL(__lib_ring_buffer_copy_to_user
);
896 * lib_ring_buffer_read_cstr - read a C-style string from ring_buffer.
897 * @bufb : buffer backend
898 * @offset : offset within the buffer
899 * @dest : destination address
900 * @len : destination's length
902 * Return string's length, or -EINVAL on error.
903 * Should be protected by get_subbuf/put_subbuf.
904 * Destination length should be at least 1 to hold '\0'.
906 int lib_ring_buffer_read_cstr(struct lib_ring_buffer_backend
*bufb
, size_t offset
,
907 void *dest
, size_t len
)
909 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
910 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
912 ssize_t pagecpy
, pagelen
, strpagelen
, orig_offset
;
914 struct lib_ring_buffer_backend_pages
*rpages
;
915 unsigned long sb_bindex
, id
;
917 offset
&= chanb
->buf_size
- 1;
918 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
919 orig_offset
= offset
;
923 id
= bufb
->buf_rsb
.id
;
924 sb_bindex
= subbuffer_id_get_index(config
, id
);
925 rpages
= bufb
->array
[sb_bindex
];
926 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
927 && subbuffer_id_is_noref(config
, id
));
928 str
= (char *)rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
);
929 pagelen
= PAGE_SIZE
- (offset
& ~PAGE_MASK
);
930 strpagelen
= strnlen(str
, pagelen
);
932 pagecpy
= min_t(size_t, len
, strpagelen
);
934 memcpy(dest
, str
, pagecpy
);
939 offset
+= strpagelen
;
940 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
941 if (strpagelen
< pagelen
)
944 * Underlying layer should never ask for reads across
947 CHAN_WARN_ON(chanb
, offset
>= chanb
->buf_size
);
950 ((char *)dest
)[0] = 0;
951 return offset
- orig_offset
;
953 EXPORT_SYMBOL_GPL(lib_ring_buffer_read_cstr
);
956 * lib_ring_buffer_read_get_page - Get a whole page to read from
957 * @bufb : buffer backend
958 * @offset : offset within the buffer
959 * @virt : pointer to page address (output)
961 * Should be protected by get_subbuf/put_subbuf.
962 * Returns the pointer to the page struct pointer.
964 struct page
**lib_ring_buffer_read_get_page(struct lib_ring_buffer_backend
*bufb
,
965 size_t offset
, void ***virt
)
968 struct lib_ring_buffer_backend_pages
*rpages
;
969 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
970 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
971 unsigned long sb_bindex
, id
;
973 offset
&= chanb
->buf_size
- 1;
974 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
975 id
= bufb
->buf_rsb
.id
;
976 sb_bindex
= subbuffer_id_get_index(config
, id
);
977 rpages
= bufb
->array
[sb_bindex
];
978 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
979 && subbuffer_id_is_noref(config
, id
));
980 *virt
= &rpages
->p
[index
].virt
;
981 return &rpages
->p
[index
].page
;
983 EXPORT_SYMBOL_GPL(lib_ring_buffer_read_get_page
);
986 * lib_ring_buffer_read_offset_address - get address of a buffer location
987 * @bufb : buffer backend
988 * @offset : offset within the buffer.
990 * Return the address where a given offset is located (for read).
991 * Should be used to get the current subbuffer header pointer. Given we know
992 * it's never on a page boundary, it's safe to read/write directly
993 * from/to this address, as long as the read/write is never bigger than a
996 void *lib_ring_buffer_read_offset_address(struct lib_ring_buffer_backend
*bufb
,
1000 struct lib_ring_buffer_backend_pages
*rpages
;
1001 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
1002 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
1003 unsigned long sb_bindex
, id
;
1005 offset
&= chanb
->buf_size
- 1;
1006 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
1007 id
= bufb
->buf_rsb
.id
;
1008 sb_bindex
= subbuffer_id_get_index(config
, id
);
1009 rpages
= bufb
->array
[sb_bindex
];
1010 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
1011 && subbuffer_id_is_noref(config
, id
));
1012 return rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
);
1014 EXPORT_SYMBOL_GPL(lib_ring_buffer_read_offset_address
);
1017 * lib_ring_buffer_offset_address - get address of a location within the buffer
1018 * @bufb : buffer backend
1019 * @offset : offset within the buffer.
1021 * Return the address where a given offset is located.
1022 * Should be used to get the current subbuffer header pointer. Given we know
1023 * it's always at the beginning of a page, it's safe to write directly to this
1024 * address, as long as the write is never bigger than a page size.
1026 void *lib_ring_buffer_offset_address(struct lib_ring_buffer_backend
*bufb
,
1029 size_t sbidx
, index
;
1030 struct lib_ring_buffer_backend_pages
*rpages
;
1031 struct channel_backend
*chanb
= &bufb
->chan
->backend
;
1032 const struct lib_ring_buffer_config
*config
= &chanb
->config
;
1033 unsigned long sb_bindex
, id
;
1035 offset
&= chanb
->buf_size
- 1;
1036 sbidx
= offset
>> chanb
->subbuf_size_order
;
1037 index
= (offset
& (chanb
->subbuf_size
- 1)) >> PAGE_SHIFT
;
1038 id
= bufb
->buf_wsb
[sbidx
].id
;
1039 sb_bindex
= subbuffer_id_get_index(config
, id
);
1040 rpages
= bufb
->array
[sb_bindex
];
1041 CHAN_WARN_ON(chanb
, config
->mode
== RING_BUFFER_OVERWRITE
1042 && subbuffer_id_is_noref(config
, id
));
1043 return rpages
->p
[index
].virt
+ (offset
& ~PAGE_MASK
);
1045 EXPORT_SYMBOL_GPL(lib_ring_buffer_offset_address
);