Fix: pass private data to context callbacks
[lttng-ust.git] / libringbuffer / shm.c
1 /*
2 * SPDX-License-Identifier: LGPL-2.1-only
3 *
4 * Copyright (C) 2005-2012 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
5 */
6
7 #define _LGPL_SOURCE
8 #include "shm.h"
9 #include <unistd.h>
10 #include <fcntl.h>
11 #include <sys/mman.h>
12 #include <sys/types.h>
13 #include <sys/stat.h> /* For mode constants */
14 #include <fcntl.h> /* For O_* constants */
15 #include <assert.h>
16 #include <stdio.h>
17 #include <signal.h>
18 #include <dirent.h>
19 #include <limits.h>
20 #include <stdbool.h>
21 #include <stdint.h>
22
23 #ifdef HAVE_LIBNUMA
24 #include <numa.h>
25 #include <numaif.h>
26 #endif
27
28 #include <lttng/ust-utils.h>
29
30 #include <ust-helper.h>
31 #include <ust-fd.h>
32 #include "mmap.h"
33
34 /*
35 * Ensure we have the required amount of space available by writing 0
36 * into the entire buffer. Not doing so can trigger SIGBUS when going
37 * beyond the available shm space.
38 */
39 static
40 int zero_file(int fd, size_t len)
41 {
42 ssize_t retlen;
43 size_t written = 0;
44 char *zeropage;
45 long pagelen;
46 int ret;
47
48 pagelen = sysconf(_SC_PAGESIZE);
49 if (pagelen < 0)
50 return (int) pagelen;
51 zeropage = calloc(pagelen, 1);
52 if (!zeropage)
53 return -ENOMEM;
54
55 while (len > written) {
56 do {
57 retlen = write(fd, zeropage,
58 min_t(size_t, pagelen, len - written));
59 } while (retlen == -1UL && errno == EINTR);
60 if (retlen < 0) {
61 ret = (int) retlen;
62 goto error;
63 }
64 written += retlen;
65 }
66 ret = 0;
67 error:
68 free(zeropage);
69 return ret;
70 }
71
72 struct shm_object_table *shm_object_table_create(size_t max_nb_obj)
73 {
74 struct shm_object_table *table;
75
76 table = zmalloc(sizeof(struct shm_object_table) +
77 max_nb_obj * sizeof(table->objects[0]));
78 if (!table)
79 return NULL;
80 table->size = max_nb_obj;
81 return table;
82 }
83
84 static
85 struct shm_object *_shm_object_table_alloc_shm(struct shm_object_table *table,
86 size_t memory_map_size,
87 int stream_fd)
88 {
89 int shmfd, waitfd[2], ret, i;
90 struct shm_object *obj;
91 char *memory_map;
92
93 if (stream_fd < 0)
94 return NULL;
95 if (table->allocated_len >= table->size)
96 return NULL;
97 obj = &table->objects[table->allocated_len];
98
99 /* wait_fd: create pipe */
100 ret = pipe(waitfd);
101 if (ret < 0) {
102 PERROR("pipe");
103 goto error_pipe;
104 }
105 for (i = 0; i < 2; i++) {
106 ret = fcntl(waitfd[i], F_SETFD, FD_CLOEXEC);
107 if (ret < 0) {
108 PERROR("fcntl");
109 goto error_fcntl;
110 }
111 }
112 /* The write end of the pipe needs to be non-blocking */
113 ret = fcntl(waitfd[1], F_SETFL, O_NONBLOCK);
114 if (ret < 0) {
115 PERROR("fcntl");
116 goto error_fcntl;
117 }
118 memcpy(obj->wait_fd, waitfd, sizeof(waitfd));
119
120 /*
121 * Set POSIX shared memory object size
122 *
123 * First, use ftruncate() to set its size, some implementations won't
124 * allow writes past the size set by ftruncate.
125 * Then, use write() to fill it with zeros, this allows us to fully
126 * allocate it and detect a shortage of shm space without dealing with
127 * a SIGBUS.
128 */
129
130 shmfd = stream_fd;
131 ret = ftruncate(shmfd, memory_map_size);
132 if (ret) {
133 PERROR("ftruncate");
134 goto error_ftruncate;
135 }
136 ret = zero_file(shmfd, memory_map_size);
137 if (ret) {
138 PERROR("zero_file");
139 goto error_zero_file;
140 }
141
142 /*
143 * Also ensure the file metadata is synced with the storage by using
144 * fsync(2). Some platforms don't allow fsync on POSIX shm fds, ignore
145 * EINVAL accordingly.
146 */
147 ret = fsync(shmfd);
148 if (ret && errno != EINVAL) {
149 PERROR("fsync");
150 goto error_fsync;
151 }
152 obj->shm_fd_ownership = 0;
153 obj->shm_fd = shmfd;
154
155 /* memory_map: mmap */
156 memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
157 MAP_SHARED | LTTNG_MAP_POPULATE, shmfd, 0);
158 if (memory_map == MAP_FAILED) {
159 PERROR("mmap");
160 goto error_mmap;
161 }
162 obj->type = SHM_OBJECT_SHM;
163 obj->memory_map = memory_map;
164 obj->memory_map_size = memory_map_size;
165 obj->allocated_len = 0;
166 obj->index = table->allocated_len++;
167
168 return obj;
169
170 error_mmap:
171 error_fsync:
172 error_ftruncate:
173 error_zero_file:
174 error_fcntl:
175 for (i = 0; i < 2; i++) {
176 ret = close(waitfd[i]);
177 if (ret) {
178 PERROR("close");
179 assert(0);
180 }
181 }
182 error_pipe:
183 return NULL;
184 }
185
186 static
187 struct shm_object *_shm_object_table_alloc_mem(struct shm_object_table *table,
188 size_t memory_map_size)
189 {
190 struct shm_object *obj;
191 void *memory_map;
192 int waitfd[2], i, ret;
193
194 if (table->allocated_len >= table->size)
195 return NULL;
196 obj = &table->objects[table->allocated_len];
197
198 memory_map = zmalloc(memory_map_size);
199 if (!memory_map)
200 goto alloc_error;
201
202 /* wait_fd: create pipe */
203 ret = pipe(waitfd);
204 if (ret < 0) {
205 PERROR("pipe");
206 goto error_pipe;
207 }
208 for (i = 0; i < 2; i++) {
209 ret = fcntl(waitfd[i], F_SETFD, FD_CLOEXEC);
210 if (ret < 0) {
211 PERROR("fcntl");
212 goto error_fcntl;
213 }
214 }
215 /* The write end of the pipe needs to be non-blocking */
216 ret = fcntl(waitfd[1], F_SETFL, O_NONBLOCK);
217 if (ret < 0) {
218 PERROR("fcntl");
219 goto error_fcntl;
220 }
221 memcpy(obj->wait_fd, waitfd, sizeof(waitfd));
222
223 /* no shm_fd */
224 obj->shm_fd = -1;
225 obj->shm_fd_ownership = 0;
226
227 obj->type = SHM_OBJECT_MEM;
228 obj->memory_map = memory_map;
229 obj->memory_map_size = memory_map_size;
230 obj->allocated_len = 0;
231 obj->index = table->allocated_len++;
232
233 return obj;
234
235 error_fcntl:
236 for (i = 0; i < 2; i++) {
237 ret = close(waitfd[i]);
238 if (ret) {
239 PERROR("close");
240 assert(0);
241 }
242 }
243 error_pipe:
244 free(memory_map);
245 alloc_error:
246 return NULL;
247 }
248
249 /*
250 * libnuma prints errors on the console even for numa_available().
251 * Work-around this limitation by using get_mempolicy() directly to
252 * check whether the kernel supports mempolicy.
253 */
254 #ifdef HAVE_LIBNUMA
255 static bool lttng_is_numa_available(void)
256 {
257 int ret;
258
259 ret = get_mempolicy(NULL, NULL, 0, NULL, 0);
260 if (ret && errno == ENOSYS) {
261 return false;
262 }
263 return numa_available() > 0;
264 }
265 #endif
266
267 struct shm_object *shm_object_table_alloc(struct shm_object_table *table,
268 size_t memory_map_size,
269 enum shm_object_type type,
270 int stream_fd,
271 int cpu)
272 {
273 struct shm_object *shm_object;
274 #ifdef HAVE_LIBNUMA
275 int oldnode = 0, node;
276 bool numa_avail;
277
278 numa_avail = lttng_is_numa_available();
279 if (numa_avail) {
280 oldnode = numa_preferred();
281 if (cpu >= 0) {
282 node = numa_node_of_cpu(cpu);
283 if (node >= 0)
284 numa_set_preferred(node);
285 }
286 if (cpu < 0 || node < 0)
287 numa_set_localalloc();
288 }
289 #endif /* HAVE_LIBNUMA */
290 switch (type) {
291 case SHM_OBJECT_SHM:
292 shm_object = _shm_object_table_alloc_shm(table, memory_map_size,
293 stream_fd);
294 break;
295 case SHM_OBJECT_MEM:
296 shm_object = _shm_object_table_alloc_mem(table, memory_map_size);
297 break;
298 default:
299 assert(0);
300 }
301 #ifdef HAVE_LIBNUMA
302 if (numa_avail)
303 numa_set_preferred(oldnode);
304 #endif /* HAVE_LIBNUMA */
305 return shm_object;
306 }
307
308 struct shm_object *shm_object_table_append_shm(struct shm_object_table *table,
309 int shm_fd, int wakeup_fd, uint32_t stream_nr,
310 size_t memory_map_size)
311 {
312 struct shm_object *obj;
313 char *memory_map;
314 int ret;
315
316 if (table->allocated_len >= table->size)
317 return NULL;
318 /* streams _must_ be received in sequential order, else fail. */
319 if (stream_nr + 1 != table->allocated_len)
320 return NULL;
321
322 obj = &table->objects[table->allocated_len];
323
324 /* wait_fd: set write end of the pipe. */
325 obj->wait_fd[0] = -1; /* read end is unset */
326 obj->wait_fd[1] = wakeup_fd;
327 obj->shm_fd = shm_fd;
328 obj->shm_fd_ownership = 1;
329
330 /* The write end of the pipe needs to be non-blocking */
331 ret = fcntl(obj->wait_fd[1], F_SETFL, O_NONBLOCK);
332 if (ret < 0) {
333 PERROR("fcntl");
334 goto error_fcntl;
335 }
336
337 /* memory_map: mmap */
338 memory_map = mmap(NULL, memory_map_size, PROT_READ | PROT_WRITE,
339 MAP_SHARED | LTTNG_MAP_POPULATE, shm_fd, 0);
340 if (memory_map == MAP_FAILED) {
341 PERROR("mmap");
342 goto error_mmap;
343 }
344 obj->type = SHM_OBJECT_SHM;
345 obj->memory_map = memory_map;
346 obj->memory_map_size = memory_map_size;
347 obj->allocated_len = memory_map_size;
348 obj->index = table->allocated_len++;
349
350 return obj;
351
352 error_fcntl:
353 error_mmap:
354 return NULL;
355 }
356
357 /*
358 * Passing ownership of mem to object.
359 */
360 struct shm_object *shm_object_table_append_mem(struct shm_object_table *table,
361 void *mem, size_t memory_map_size, int wakeup_fd)
362 {
363 struct shm_object *obj;
364 int ret;
365
366 if (table->allocated_len >= table->size)
367 return NULL;
368 obj = &table->objects[table->allocated_len];
369
370 obj->wait_fd[0] = -1; /* read end is unset */
371 obj->wait_fd[1] = wakeup_fd;
372 obj->shm_fd = -1;
373 obj->shm_fd_ownership = 0;
374
375 ret = fcntl(obj->wait_fd[1], F_SETFD, FD_CLOEXEC);
376 if (ret < 0) {
377 PERROR("fcntl");
378 goto error_fcntl;
379 }
380 /* The write end of the pipe needs to be non-blocking */
381 ret = fcntl(obj->wait_fd[1], F_SETFL, O_NONBLOCK);
382 if (ret < 0) {
383 PERROR("fcntl");
384 goto error_fcntl;
385 }
386
387 obj->type = SHM_OBJECT_MEM;
388 obj->memory_map = mem;
389 obj->memory_map_size = memory_map_size;
390 obj->allocated_len = memory_map_size;
391 obj->index = table->allocated_len++;
392
393 return obj;
394
395 error_fcntl:
396 return NULL;
397 }
398
399 static
400 void shmp_object_destroy(struct shm_object *obj, int consumer)
401 {
402 switch (obj->type) {
403 case SHM_OBJECT_SHM:
404 {
405 int ret, i;
406
407 ret = munmap(obj->memory_map, obj->memory_map_size);
408 if (ret) {
409 PERROR("umnmap");
410 assert(0);
411 }
412
413 if (obj->shm_fd_ownership) {
414 /* Delete FDs only if called from app (not consumer). */
415 if (!consumer) {
416 lttng_ust_lock_fd_tracker();
417 ret = close(obj->shm_fd);
418 if (!ret) {
419 lttng_ust_delete_fd_from_tracker(obj->shm_fd);
420 } else {
421 PERROR("close");
422 assert(0);
423 }
424 lttng_ust_unlock_fd_tracker();
425 } else {
426 ret = close(obj->shm_fd);
427 if (ret) {
428 PERROR("close");
429 assert(0);
430 }
431 }
432 }
433 for (i = 0; i < 2; i++) {
434 if (obj->wait_fd[i] < 0)
435 continue;
436 if (!consumer) {
437 lttng_ust_lock_fd_tracker();
438 ret = close(obj->wait_fd[i]);
439 if (!ret) {
440 lttng_ust_delete_fd_from_tracker(obj->wait_fd[i]);
441 } else {
442 PERROR("close");
443 assert(0);
444 }
445 lttng_ust_unlock_fd_tracker();
446 } else {
447 ret = close(obj->wait_fd[i]);
448 if (ret) {
449 PERROR("close");
450 assert(0);
451 }
452 }
453 }
454 break;
455 }
456 case SHM_OBJECT_MEM:
457 {
458 int ret, i;
459
460 for (i = 0; i < 2; i++) {
461 if (obj->wait_fd[i] < 0)
462 continue;
463 if (!consumer) {
464 lttng_ust_lock_fd_tracker();
465 ret = close(obj->wait_fd[i]);
466 if (!ret) {
467 lttng_ust_delete_fd_from_tracker(obj->wait_fd[i]);
468 } else {
469 PERROR("close");
470 assert(0);
471 }
472 lttng_ust_unlock_fd_tracker();
473 } else {
474 ret = close(obj->wait_fd[i]);
475 if (ret) {
476 PERROR("close");
477 assert(0);
478 }
479 }
480 }
481 free(obj->memory_map);
482 break;
483 }
484 default:
485 assert(0);
486 }
487 }
488
489 void shm_object_table_destroy(struct shm_object_table *table, int consumer)
490 {
491 int i;
492
493 for (i = 0; i < table->allocated_len; i++)
494 shmp_object_destroy(&table->objects[i], consumer);
495 free(table);
496 }
497
498 /*
499 * zalloc_shm - allocate memory within a shm object.
500 *
501 * Shared memory is already zeroed by shmget.
502 * *NOT* multithread-safe (should be protected by mutex).
503 * Returns a -1, -1 tuple on error.
504 */
505 struct shm_ref zalloc_shm(struct shm_object *obj, size_t len)
506 {
507 struct shm_ref ref;
508 struct shm_ref shm_ref_error = { -1, -1 };
509
510 if (obj->memory_map_size - obj->allocated_len < len)
511 return shm_ref_error;
512 ref.index = obj->index;
513 ref.offset = obj->allocated_len;
514 obj->allocated_len += len;
515 return ref;
516 }
517
518 void align_shm(struct shm_object *obj, size_t align)
519 {
520 size_t offset_len = lttng_ust_offset_align(obj->allocated_len, align);
521 obj->allocated_len += offset_len;
522 }
This page took 0.042501 seconds and 4 git commands to generate.