src/bin/lttng-sessiond/rotation-thread.c

   1 /*
   2  * Copyright (C) 2017 - Julien Desfossez <jdesfossez@efficios.com>
   3  * Copyright (C) 2018 - Jérémie Galarneau <jeremie.galarneau@efficios.com>
   4  *
   5  * This program is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License, version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
  12  * more details.
  13  *
  14  * You should have received a copy of the GNU General Public License along with
  15  * this program; if not, write to the Free Software Foundation, Inc., 51
  16  * Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  17  */
  18
  19 #define _LGPL_SOURCE
  20 #include <lttng/trigger/trigger.h>
  21 #include <common/error.h>
  22 #include <common/config/session-config.h>
  23 #include <common/defaults.h>
  24 #include <common/utils.h>
  25 #include <common/futex.h>
  26 #include <common/align.h>
  27 #include <common/time.h>
  28 #include <common/hashtable/utils.h>
  29 #include <sys/eventfd.h>
  30 #include <sys/stat.h>
  31 #include <time.h>
  32 #include <signal.h>
  33 #include <inttypes.h>
  34
  35 #include <common/kernel-ctl/kernel-ctl.h>
  36 #include <lttng/notification/channel-internal.h>
  37 #include <lttng/rotate-internal.h>
  38
  39 #include "rotation-thread.h"
  40 #include "lttng-sessiond.h"
  41 #include "health-sessiond.h"
  42 #include "rotate.h"
  43 #include "cmd.h"
  44 #include "session.h"
  45 #include "timer.h"
  46 #include "notification-thread-commands.h"
  47 #include "utils.h"
  48 #include "thread.h"
  49
  50 #include <urcu.h>
  51 #include <urcu/list.h>
  52
  53 struct lttng_notification_channel *rotate_notification_channel = NULL;
  54
  55 struct rotation_thread {
  56         struct lttng_poll_event events;
  57 };
  58
  59 struct rotation_thread_job {
  60         enum rotation_thread_job_type type;
  61         struct ltt_session *session;
  62         /* List member in struct rotation_thread_timer_queue. */
  63         struct cds_list_head head;
  64 };
  65
  66 /*
  67  * The timer thread enqueues jobs and wakes up the rotation thread.
  68  * When the rotation thread wakes up, it empties the queue.
  69  */
  70 struct rotation_thread_timer_queue {
  71         struct lttng_pipe *event_pipe;
  72         struct cds_list_head list;
  73         pthread_mutex_t lock;
  74 };
  75
  76 struct rotation_thread_handle {
  77         struct rotation_thread_timer_queue *rotation_timer_queue;
  78         /* Access to the notification thread cmd_queue */
  79         struct notification_thread_handle *notification_thread_handle;
  80         /* Thread-specific quit pipe. */
  81         struct lttng_pipe *quit_pipe;
  82 };
  83
  84 static
  85 const char *get_job_type_str(enum rotation_thread_job_type job_type)
  86 {
  87         switch (job_type) {
  88         case ROTATION_THREAD_JOB_TYPE_CHECK_PENDING_ROTATION:
  89                 return "CHECK_PENDING_ROTATION";
  90         case ROTATION_THREAD_JOB_TYPE_SCHEDULED_ROTATION:
  91                 return "SCHEDULED_ROTATION";
  92         default:
  93                 abort();
  94         }
  95 }
  96
  97 struct rotation_thread_timer_queue *rotation_thread_timer_queue_create(void)
  98 {
  99         struct rotation_thread_timer_queue *queue = NULL;
 100
 101         queue = zmalloc(sizeof(*queue));
 102         if (!queue) {
 103                 PERROR("Failed to allocate timer rotate queue");
 104                 goto end;
 105         }
 106
 107         queue->event_pipe = lttng_pipe_open(FD_CLOEXEC | O_NONBLOCK);
 108         CDS_INIT_LIST_HEAD(&queue->list);
 109         pthread_mutex_init(&queue->lock, NULL);
 110 end:
 111         return queue;
 112 }
 113
 114 void log_job_destruction(const struct rotation_thread_job *job)
 115 {
 116         enum lttng_error_level log_level;
 117         const char *job_type_str = get_job_type_str(job->type);
 118
 119         switch (job->type) {
 120         case ROTATION_THREAD_JOB_TYPE_SCHEDULED_ROTATION:
 121                 /*
 122                  * Not a problem, the scheduled rotation is racing with the teardown
 123                  * of the daemon. In this case, the rotation will not happen, which
 124                  * is not a problem (or at least, not important enough to delay
 125                  * the shutdown of the session daemon).
 126                  */
 127                 log_level = PRINT_DBG;
 128                 break;
 129         case ROTATION_THREAD_JOB_TYPE_CHECK_PENDING_ROTATION:
 130                 /* This is not expected to happen; warn the user. */
 131                 log_level = PRINT_WARN;
 132                 break;
 133         default:
 134                 abort();
 135         }
 136
 137         LOG(log_level, "Rotation thread timer queue still contains job of type %s targeting session \"%s\" on destruction",
 138                         job_type_str, job->session->name);
 139 }
 140
 141 void rotation_thread_timer_queue_destroy(
 142                 struct rotation_thread_timer_queue *queue)
 143 {
 144         if (!queue) {
 145                 return;
 146         }
 147
 148         lttng_pipe_destroy(queue->event_pipe);
 149
 150         pthread_mutex_lock(&queue->lock);
 151         assert(cds_list_empty(&queue->list));
 152         pthread_mutex_unlock(&queue->lock);
 153         pthread_mutex_destroy(&queue->lock);
 154         free(queue);
 155 }
 156
 157 /*
 158  * Destroy the thread data previously created by the init function.
 159  */
 160 void rotation_thread_handle_destroy(
 161                 struct rotation_thread_handle *handle)
 162 {
 163         lttng_pipe_destroy(handle->quit_pipe);
 164         free(handle);
 165 }
 166
 167 struct rotation_thread_handle *rotation_thread_handle_create(
 168                 struct rotation_thread_timer_queue *rotation_timer_queue,
 169                 struct notification_thread_handle *notification_thread_handle)
 170 {
 171         struct rotation_thread_handle *handle;
 172
 173         handle = zmalloc(sizeof(*handle));
 174         if (!handle) {
 175                 goto end;
 176         }
 177
 178         handle->rotation_timer_queue = rotation_timer_queue;
 179         handle->notification_thread_handle = notification_thread_handle;
 180         handle->quit_pipe = lttng_pipe_open(FD_CLOEXEC);
 181         if (!handle->quit_pipe) {
 182                 goto error;
 183         }
 184
 185 end:
 186         return handle;
 187 error:
 188         rotation_thread_handle_destroy(handle);
 189         return NULL;
 190 }
 191
 192 /*
 193  * Called with the rotation_thread_timer_queue lock held.
 194  * Return true if the same timer job already exists in the queue, false if not.
 195  */
 196 static
 197 bool timer_job_exists(const struct rotation_thread_timer_queue *queue,
 198                 enum rotation_thread_job_type job_type,
 199                 struct ltt_session *session)
 200 {
 201         bool exists = false;
 202         struct rotation_thread_job *job;
 203
 204         cds_list_for_each_entry(job, &queue->list, head) {
 205                 if (job->session == session && job->type == job_type) {
 206                         exists = true;
 207                         goto end;
 208                 }
 209         }
 210 end:
 211         return exists;
 212 }
 213
 214 void rotation_thread_enqueue_job(struct rotation_thread_timer_queue *queue,
 215                 enum rotation_thread_job_type job_type,
 216                 struct ltt_session *session)
 217 {
 218         int ret;
 219         const char * const dummy = "!";
 220         struct rotation_thread_job *job = NULL;
 221         const char *job_type_str = get_job_type_str(job_type);
 222
 223         pthread_mutex_lock(&queue->lock);
 224         if (timer_job_exists(queue, job_type, session)) {
 225                 /*
 226                  * This timer job is already pending, we don't need to add
 227                  * it.
 228                  */
 229                 goto end;
 230         }
 231
 232         job = zmalloc(sizeof(struct rotation_thread_job));
 233         if (!job) {
 234                 PERROR("Failed to allocate rotation thread job of type \"%s\" for session \"%s\"",
 235                                 job_type_str, session->name);
 236                 goto end;
 237         }
 238         /* No reason for this to fail as the caller must hold a reference. */
 239         (void) session_get(session);
 240
 241         job->session = session;
 242         job->type = job_type;
 243         cds_list_add_tail(&job->head, &queue->list);
 244
 245         ret = lttng_write(lttng_pipe_get_writefd(queue->event_pipe), dummy,
 246                         1);
 247         if (ret < 0) {
 248                 /*
 249                  * We do not want to block in the timer handler, the job has
 250                  * been enqueued in the list, the wakeup pipe is probably full,
 251                  * the job will be processed when the rotation_thread catches
 252                  * up.
 253                  */
 254                 if (errno == EAGAIN || errno == EWOULDBLOCK) {
 255                         /*
 256                          * Not an error, but would be surprising and indicate
 257                          * that the rotation thread can't keep up with the
 258                          * current load.
 259                          */
 260                         DBG("Wake-up pipe of rotation thread job queue is full");
 261                         goto end;
 262                 }
 263                 PERROR("Failed to wake-up the rotation thread after pushing a job of type \"%s\" for session \"%s\"",
 264                                 job_type_str, session->name);
 265                 goto end;
 266         }
 267
 268 end:
 269         pthread_mutex_unlock(&queue->lock);
 270 }
 271
 272 static
 273 int init_poll_set(struct lttng_poll_event *poll_set,
 274                 struct rotation_thread_handle *handle)
 275 {
 276         int ret;
 277
 278         /*
 279          * Create pollset with size 3:
 280          *      - rotation thread quit pipe,
 281          *      - rotation thread timer queue pipe,
 282          *      - notification channel sock,
 283          */
 284         ret = lttng_poll_create(poll_set, 5, LTTNG_CLOEXEC);
 285         if (ret < 0) {
 286                 goto error;
 287         }
 288
 289         ret = lttng_poll_add(poll_set,
 290                         lttng_pipe_get_readfd(handle->quit_pipe),
 291                         LPOLLIN | LPOLLERR);
 292         if (ret < 0) {
 293                 ERR("[rotation-thread] Failed to add quit pipe read fd to poll set");
 294                 goto error;
 295         }
 296
 297         ret = lttng_poll_add(poll_set,
 298                         lttng_pipe_get_readfd(handle->rotation_timer_queue->event_pipe),
 299                         LPOLLIN | LPOLLERR);
 300         if (ret < 0) {
 301                 ERR("[rotation-thread] Failed to add rotate_pending fd to poll set");
 302                 goto error;
 303         }
 304
 305         return ret;
 306 error:
 307         lttng_poll_clean(poll_set);
 308         return ret;
 309 }
 310
 311 static
 312 void fini_thread_state(struct rotation_thread *state)
 313 {
 314         lttng_poll_clean(&state->events);
 315         if (rotate_notification_channel) {
 316                 lttng_notification_channel_destroy(rotate_notification_channel);
 317         }
 318 }
 319
 320 static
 321 int init_thread_state(struct rotation_thread_handle *handle,
 322                 struct rotation_thread *state)
 323 {
 324         int ret;
 325
 326         memset(state, 0, sizeof(*state));
 327         lttng_poll_init(&state->events);
 328
 329         ret = init_poll_set(&state->events, handle);
 330         if (ret) {
 331                 ERR("[rotation-thread] Failed to initialize rotation thread poll set");
 332                 goto end;
 333         }
 334
 335         rotate_notification_channel = lttng_notification_channel_create(
 336                         lttng_session_daemon_notification_endpoint);
 337         if (!rotate_notification_channel) {
 338                 ERR("[rotation-thread] Could not create notification channel");
 339                 ret = -1;
 340                 goto end;
 341         }
 342         ret = lttng_poll_add(&state->events, rotate_notification_channel->socket,
 343                         LPOLLIN | LPOLLERR);
 344         if (ret < 0) {
 345                 ERR("[rotation-thread] Failed to add notification fd to pollset");
 346                 goto end;
 347         }
 348
 349 end:
 350         return ret;
 351 }
 352
 353 static
 354 int check_session_rotation_pending_local_on_consumer(
 355                 const struct ltt_session *session,
 356                 struct consumer_socket *socket, bool *rotation_completed)
 357 {
 358         int ret;
 359
 360         pthread_mutex_lock(socket->lock);
 361         DBG("[rotation-thread] Checking for locally pending rotation on the %s consumer for session %s",
 362                         lttng_consumer_type_str(socket->type),
 363                         session->name);
 364         ret = consumer_check_rotation_pending_local(socket,
 365                         session->id,
 366                         session->current_archive_id - 1);
 367         pthread_mutex_unlock(socket->lock);
 368
 369         if (ret == 0) {
 370                 /* Rotation was completed on this consumer. */
 371                 DBG("[rotation-thread] Local rotation of trace archive %" PRIu64 " of session \"%s\" was completed on the %s consumer",
 372                                 session->current_archive_id - 1,
 373                                 session->name,
 374                                 lttng_consumer_type_str(socket->type));
 375                 *rotation_completed = true;
 376         } else if (ret == 1) {
 377                 /* Rotation pending on this consumer. */
 378                 DBG("[rotation-thread] Local rotation of trace archive %" PRIu64 " of session \"%s\" is pending on the %s consumer",
 379                                 session->current_archive_id - 1,
 380                                 session->name,
 381                                 lttng_consumer_type_str(socket->type));
 382                 *rotation_completed = false;
 383                 ret = 0;
 384         } else {
 385                 /* Not a fatal error. */
 386                 ERR("[rotation-thread] Encountered an error when checking if local rotation of trace archive %" PRIu64 " of session \"%s\" is pending on the %s consumer",
 387                                 session->current_archive_id - 1,
 388                                 session->name,
 389                                 lttng_consumer_type_str(socket->type));
 390                 *rotation_completed = false;
 391         }
 392         return ret;
 393 }
 394
 395 static
 396 int check_session_rotation_pending_local(struct ltt_session *session)
 397 {
 398         int ret = 0;
 399         struct consumer_socket *socket;
 400         struct cds_lfht_iter iter;
 401         bool rotation_completed = true;
 402
 403         /*
 404          * Check for a local pending rotation on all consumers (32-bit
 405          * user space, 64-bit user space, and kernel).
 406          */
 407         DBG("[rotation-thread] Checking for pending local rotation on session \"%s\", trace archive %" PRIu64,
 408                         session->name, session->current_archive_id - 1);
 409
 410         rcu_read_lock();
 411         if (!session->ust_session) {
 412                 goto skip_ust;
 413         }
 414         cds_lfht_for_each_entry(session->ust_session->consumer->socks->ht,
 415                         &iter, socket, node.node) {
 416                 ret = check_session_rotation_pending_local_on_consumer(session,
 417                                 socket, &rotation_completed);
 418                 if (ret || !rotation_completed) {
 419                         goto end;
 420                 }
 421         }
 422
 423 skip_ust:
 424         if (!session->kernel_session) {
 425                 goto skip_kernel;
 426         }
 427         cds_lfht_for_each_entry(session->kernel_session->consumer->socks->ht,
 428                                 &iter, socket, node.node) {
 429                 ret = check_session_rotation_pending_local_on_consumer(session,
 430                                 socket, &rotation_completed);
 431                 if (ret || !rotation_completed) {
 432                         goto end;
 433                 }
 434         }
 435 skip_kernel:
 436 end:
 437         rcu_read_unlock();
 438
 439         if (rotation_completed) {
 440                 DBG("[rotation-thread] Local rotation of trace archive %" PRIu64 " of session \"%s\" is complete on all consumers",
 441                                 session->current_archive_id - 1,
 442                                 session->name);
 443                 session->rotation_pending_local = false;
 444         }
 445         if (ret) {
 446                 ret = session_reset_rotation_state(session,
 447                                 LTTNG_ROTATION_STATE_ERROR);
 448                 if (ret) {
 449                         ERR("Failed to reset rotation state of session \"%s\"",
 450                                         session->name);
 451                 }
 452         }
 453         return 0;
 454 }
 455
 456 static
 457 int check_session_rotation_pending_relay(struct ltt_session *session)
 458 {
 459         int ret;
 460         struct consumer_socket *socket;
 461         struct cds_lfht_iter iter;
 462         bool rotation_completed = true;
 463         const struct consumer_output *output;
 464
 465         /*
 466          * Check for a pending rotation on any consumer as we only use
 467          * it as a "tunnel" to the relayd.
 468          */
 469
 470         rcu_read_lock();
 471         if (session->ust_session) {
 472                 cds_lfht_first(session->ust_session->consumer->socks->ht,
 473                                 &iter);
 474                 output = session->ust_session->consumer;
 475         } else {
 476                 cds_lfht_first(session->kernel_session->consumer->socks->ht,
 477                                 &iter);
 478                 output = session->kernel_session->consumer;
 479         }
 480         assert(cds_lfht_iter_get_node(&iter));
 481
 482         socket = caa_container_of(cds_lfht_iter_get_node(&iter),
 483                         typeof(*socket), node.node);
 484
 485         pthread_mutex_lock(socket->lock);
 486         DBG("[rotation-thread] Checking for pending relay rotation on session \"%s\", trace archive %" PRIu64 " through the %s consumer",
 487                         session->name, session->current_archive_id - 1,
 488                         lttng_consumer_type_str(socket->type));
 489         ret = consumer_check_rotation_pending_relay(socket,
 490                         output,
 491                         session->id,
 492                         session->current_archive_id - 1);
 493         pthread_mutex_unlock(socket->lock);
 494
 495         if (ret == 0) {
 496                 /* Rotation was completed on the relay. */
 497                 DBG("[rotation-thread] Relay rotation of trace archive %" PRIu64 " of session \"%s\" was completed",
 498                                 session->current_archive_id - 1,
 499                                 session->name);
 500         } else if (ret == 1) {
 501                 /* Rotation pending on relay. */
 502                 DBG("[rotation-thread] Relay rotation of trace archive %" PRIu64 " of session \"%s\" is pending",
 503                                 session->current_archive_id - 1,
 504                                 session->name);
 505                 rotation_completed = false;
 506         } else {
 507                 /* Not a fatal error. */
 508                 ERR("[rotation-thread] Encountered an error when checking if rotation of trace archive %" PRIu64 " of session \"%s\" is pending on the relay",
 509                                 session->current_archive_id - 1,
 510                                 session->name);
 511                 ret = session_reset_rotation_state(session,
 512                                 LTTNG_ROTATION_STATE_ERROR);
 513                 if (ret) {
 514                         ERR("Failed to reset rotation state of session \"%s\"",
 515                                         session->name);
 516                 }
 517                 rotation_completed = false;
 518         }
 519
 520         rcu_read_unlock();
 521
 522         if (rotation_completed) {
 523                 DBG("[rotation-thread] Rotation of trace archive %" PRIu64 " of session \"%s\" is complete on the relay",
 524                                 session->current_archive_id - 1,
 525                                 session->name);
 526                 session->rotation_pending_relay = false;
 527         }
 528         return 0;
 529 }
 530
 531 /*
 532  * Check if the last rotation was completed, called with session lock held.
 533  */
 534 static
 535 int check_session_rotation_pending(struct ltt_session *session,
 536                 struct notification_thread_handle *notification_thread_handle)
 537 {
 538         int ret;
 539         struct lttng_trace_archive_location *location;
 540         time_t now;
 541
 542         DBG("[rotation-thread] Checking for pending rotation on session \"%s\", trace archive %" PRIu64,
 543                         session->name, session->current_archive_id - 1);
 544
 545         /*
 546          * The rotation-pending check timer of a session is launched in
 547          * one-shot mode. If the rotation is incomplete, the rotation
 548          * thread will re-enable the pending-check timer.
 549          *
 550          * The timer thread can't stop the timer itself since it is involved
 551          * in the check for the timer's quiescence.
 552          */
 553         ret = timer_session_rotation_pending_check_stop(session);
 554         if (ret) {
 555                 goto end;
 556         }
 557
 558         if (session->rotation_pending_local) {
 559                 /* Updates session->rotation_pending_local as needed. */
 560                 ret = check_session_rotation_pending_local(session);
 561                 if (ret) {
 562                         goto end;
 563                 }
 564
 565                 /*
 566                  * No need to check for a pending rotation on the relay
 567                  * since the rotation is not even completed locally yet.
 568                  */
 569                 if (session->rotation_pending_local) {
 570                         goto end;
 571                 }
 572         }
 573
 574         if (session->rotation_pending_relay) {
 575                 /* Updates session->rotation_pending_relay as needed. */
 576                 ret = check_session_rotation_pending_relay(session);
 577                 if (ret) {
 578                         goto end;
 579                 }
 580
 581                 if (session->rotation_pending_relay) {
 582                         goto end;
 583                 }
 584         }
 585
 586         DBG("[rotation-thread] Rotation of trace archive %" PRIu64 " completed for "
 587                         "session %s", session->current_archive_id - 1,
 588                         session->name);
 589
 590         /* Rename the completed trace archive's location. */
 591         now = time(NULL);
 592         if (now == (time_t) -1) {
 593                 ret = session_reset_rotation_state(session,
 594                                 LTTNG_ROTATION_STATE_ERROR);
 595                 if (ret) {
 596                         ERR("Failed to reset rotation state of session \"%s\"",
 597                                         session->name);
 598                 }
 599                 ret = LTTNG_ERR_UNK;
 600                 goto end;
 601         }
 602
 603         ret = rename_completed_chunk(session, now);
 604         if (ret < 0) {
 605                 ERR("Failed to rename completed rotation chunk");
 606                 goto end;
 607         }
 608         session->last_chunk_start_ts = session->current_chunk_start_ts;
 609
 610         /*
 611          * Now we can clear the "ONGOING" state in the session. New
 612          * rotations can start now.
 613          */
 614         session->rotation_state = LTTNG_ROTATION_STATE_COMPLETED;
 615
 616         /* Ownership of location is transferred. */
 617         location = session_get_trace_archive_location(session);
 618         ret = notification_thread_command_session_rotation_completed(
 619                         notification_thread_handle,
 620                         session->name,
 621                         session->uid,
 622                         session->gid,
 623                         session->current_archive_id,
 624                         location);
 625         if (ret != LTTNG_OK) {
 626                 ERR("[rotation-thread] Failed to notify notification thread of completed rotation for session %s",
 627                                 session->name);
 628         }
 629
 630         if (!session->active) {
 631                 /*
 632                  * A stop command was issued during the rotation, it is
 633                  * up to the rotation completion check to perform the
 634                  * renaming of the last chunk that was produced.
 635                  */
 636                 ret = notification_thread_command_session_rotation_ongoing(
 637                                 notification_thread_handle,
 638                                 session->name,
 639                                 session->uid,
 640                                 session->gid,
 641                                 session->current_archive_id);
 642                 if (ret != LTTNG_OK) {
 643                         ERR("[rotation-thread] Failed to notify notification thread of completed rotation for session %s",
 644                                         session->name);
 645                 }
 646
 647                 ret = rename_active_chunk(session);
 648                 if (ret < 0) {
 649                         ERR("[rotation-thread] Failed to rename active rotation chunk");
 650                         goto end;
 651                 }
 652
 653                 /* Ownership of location is transferred. */
 654                 location = session_get_trace_archive_location(session);
 655                 ret = notification_thread_command_session_rotation_completed(
 656                                 notification_thread_handle,
 657                                 session->name,
 658                                 session->uid,
 659                                 session->gid,
 660                                 session->current_archive_id,
 661                                 location);
 662                 if (ret != LTTNG_OK) {
 663                         ERR("[rotation-thread] Failed to notify notification thread of completed rotation for session %s",
 664                                         session->name);
 665                 }
 666         }
 667
 668         ret = 0;
 669 end:
 670         if (session->rotation_state == LTTNG_ROTATION_STATE_ONGOING) {
 671                 DBG("[rotation-thread] Rotation of trace archive %" PRIu64 " is still pending for session %s",
 672                                 session->current_archive_id - 1, session->name);
 673                 ret = timer_session_rotation_pending_check_start(session,
 674                                 DEFAULT_ROTATE_PENDING_TIMER);
 675                 if (ret) {
 676                         ERR("Re-enabling rotate pending timer");
 677                         ret = -1;
 678                         goto end;
 679                 }
 680         }
 681
 682         return ret;
 683 }
 684
 685 /* Call with the session and session_list locks held. */
 686 static
 687 int launch_session_rotation(struct ltt_session *session)
 688 {
 689         int ret;
 690         struct lttng_rotate_session_return rotation_return;
 691
 692         DBG("[rotation-thread] Launching scheduled time-based rotation on session \"%s\"",
 693                         session->name);
 694
 695         ret = cmd_rotate_session(session, &rotation_return);
 696         if (ret == LTTNG_OK) {
 697                 DBG("[rotation-thread] Scheduled time-based rotation successfully launched on session \"%s\"",
 698                                 session->name);
 699         } else {
 700                 /* Don't consider errors as fatal. */
 701                 DBG("[rotation-thread] Scheduled time-based rotation aborted for session %s: %s",
 702                                 session->name, lttng_strerror(ret));
 703         }
 704         return 0;
 705 }
 706
 707 static
 708 int run_job(struct rotation_thread_job *job, struct ltt_session *session,
 709                 struct notification_thread_handle *notification_thread_handle)
 710 {
 711         int ret;
 712
 713         switch (job->type) {
 714         case ROTATION_THREAD_JOB_TYPE_SCHEDULED_ROTATION:
 715                 ret = launch_session_rotation(session);
 716                 break;
 717         case ROTATION_THREAD_JOB_TYPE_CHECK_PENDING_ROTATION:
 718                 ret = check_session_rotation_pending(session,
 719                                 notification_thread_handle);
 720                 break;
 721         default:
 722                 abort();
 723         }
 724         return ret;
 725 }
 726
 727 static
 728 int handle_job_queue(struct rotation_thread_handle *handle,
 729                 struct rotation_thread *state,
 730                 struct rotation_thread_timer_queue *queue)
 731 {
 732         int ret = 0;
 733
 734         for (;;) {
 735                 struct ltt_session *session;
 736                 struct rotation_thread_job *job;
 737
 738                 /* Take the queue lock only to pop an element from the list. */
 739                 pthread_mutex_lock(&queue->lock);
 740                 if (cds_list_empty(&queue->list)) {
 741                         pthread_mutex_unlock(&queue->lock);
 742                         break;
 743                 }
 744                 job = cds_list_first_entry(&queue->list,
 745                                 typeof(*job), head);
 746                 cds_list_del(&job->head);
 747                 pthread_mutex_unlock(&queue->lock);
 748
 749                 session_lock_list();
 750                 session = job->session;
 751                 if (!session) {
 752                         DBG("[rotation-thread] Session \"%s\" not found",
 753                                         session->name);
 754                         /*
 755                          * This is a non-fatal error, and we cannot report it to
 756                          * the user (timer), so just print the error and
 757                          * continue the processing.
 758                          *
 759                          * While the timer thread will purge pending signals for
 760                          * a session on the session's destruction, it is
 761                          * possible for a job targeting that session to have
 762                          * already been queued before it was destroyed.
 763                          */
 764                         session_unlock_list();
 765                         free(job);
 766                         session_put(session);
 767                         continue;
 768                 }
 769
 770                 session_lock(session);
 771                 ret = run_job(job, session, handle->notification_thread_handle);
 772                 session_unlock(session);
 773                 /* Release reference held by the job. */
 774                 session_put(session);
 775                 session_unlock_list();
 776                 free(job);
 777                 if (ret) {
 778                         goto end;
 779                 }
 780         }
 781
 782         ret = 0;
 783
 784 end:
 785         return ret;
 786 }
 787
 788 static
 789 int handle_condition(const struct lttng_condition *condition,
 790                 const struct lttng_evaluation *evaluation,
 791                 struct notification_thread_handle *notification_thread_handle)
 792 {
 793         int ret = 0;
 794         const char *condition_session_name = NULL;
 795         enum lttng_condition_type condition_type;
 796         enum lttng_condition_status condition_status;
 797         enum lttng_evaluation_status evaluation_status;
 798         uint64_t consumed;
 799         struct ltt_session *session;
 800
 801         condition_type = lttng_condition_get_type(condition);
 802
 803         if (condition_type != LTTNG_CONDITION_TYPE_SESSION_CONSUMED_SIZE) {
 804                 ret = -1;
 805                 ERR("[rotation-thread] Condition type and session usage type are not the same");
 806                 goto end;
 807         }
 808
 809         /* Fetch info to test */
 810         condition_status = lttng_condition_session_consumed_size_get_session_name(
 811                         condition, &condition_session_name);
 812         if (condition_status != LTTNG_CONDITION_STATUS_OK) {
 813                 ERR("[rotation-thread] Session name could not be fetched");
 814                 ret = -1;
 815                 goto end;
 816         }
 817         evaluation_status = lttng_evaluation_session_consumed_size_get_consumed_size(evaluation,
 818                         &consumed);
 819         if (evaluation_status != LTTNG_EVALUATION_STATUS_OK) {
 820                 ERR("[rotation-thread] Failed to get evaluation");
 821                 ret = -1;
 822                 goto end;
 823         }
 824
 825         session_lock_list();
 826         session = session_find_by_name(condition_session_name);
 827         if (!session) {
 828                 ret = -1;
 829                 session_unlock_list();
 830                 ERR("[rotation-thread] Session \"%s\" not found",
 831                                 condition_session_name);
 832                 goto end;
 833         }
 834         session_lock(session);
 835         session_unlock_list();
 836
 837         ret = unsubscribe_session_consumed_size_rotation(session,
 838                         notification_thread_handle);
 839         if (ret) {
 840                 goto end_unlock;
 841         }
 842
 843         ret = cmd_rotate_session(session, NULL);
 844         if (ret == -LTTNG_ERR_ROTATION_PENDING) {
 845                 DBG("Rotate already pending, subscribe to the next threshold value");
 846         } else if (ret != LTTNG_OK) {
 847                 ERR("[rotation-thread] Failed to rotate on size notification with error: %s",
 848                                 lttng_strerror(ret));
 849                 ret = -1;
 850                 goto end_unlock;
 851         }
 852         ret = subscribe_session_consumed_size_rotation(session,
 853                         consumed + session->rotate_size,
 854                         notification_thread_handle);
 855         if (ret) {
 856                 ERR("[rotation-thread] Failed to subscribe to session consumed size condition");
 857                 goto end_unlock;
 858         }
 859         ret = 0;
 860
 861 end_unlock:
 862         session_unlock(session);
 863         session_put(session);
 864 end:
 865         return ret;
 866 }
 867
 868 static
 869 int handle_notification_channel(int fd,
 870                 struct rotation_thread_handle *handle,
 871                 struct rotation_thread *state)
 872 {
 873         int ret;
 874         bool notification_pending;
 875         struct lttng_notification *notification = NULL;
 876         enum lttng_notification_channel_status status;
 877         const struct lttng_evaluation *notification_evaluation;
 878         const struct lttng_condition *notification_condition;
 879
 880         status = lttng_notification_channel_has_pending_notification(
 881                         rotate_notification_channel, &notification_pending);
 882         if (status != LTTNG_NOTIFICATION_CHANNEL_STATUS_OK) {
 883                 ERR("[rotation-thread ]Error occurred while checking for pending notification");
 884                 ret = -1;
 885                 goto end;
 886         }
 887
 888         if (!notification_pending) {
 889                 ret = 0;
 890                 goto end;
 891         }
 892
 893         /* Receive the next notification. */
 894         status = lttng_notification_channel_get_next_notification(
 895                         rotate_notification_channel,
 896                         &notification);
 897
 898         switch (status) {
 899         case LTTNG_NOTIFICATION_CHANNEL_STATUS_OK:
 900                 break;
 901         case LTTNG_NOTIFICATION_CHANNEL_STATUS_NOTIFICATIONS_DROPPED:
 902                 /* Not an error, we will wait for the next one */
 903                 ret = 0;
 904                 goto end;;
 905         case LTTNG_NOTIFICATION_CHANNEL_STATUS_CLOSED:
 906                 ERR("Notification channel was closed");
 907                 ret = -1;
 908                 goto end;
 909         default:
 910                 /* Unhandled conditions / errors. */
 911                 ERR("Unknown notification channel status");
 912                 ret = -1;
 913                 goto end;
 914         }
 915
 916         notification_condition = lttng_notification_get_condition(notification);
 917         notification_evaluation = lttng_notification_get_evaluation(notification);
 918
 919         ret = handle_condition(notification_condition, notification_evaluation,
 920                         handle->notification_thread_handle);
 921
 922 end:
 923         lttng_notification_destroy(notification);
 924         return ret;
 925 }
 926
 927 void *thread_rotation(void *data)
 928 {
 929         int ret;
 930         struct rotation_thread_handle *handle = data;
 931         struct rotation_thread thread;
 932         const int queue_pipe_fd = lttng_pipe_get_readfd(
 933                         handle->rotation_timer_queue->event_pipe);
 934
 935         DBG("[rotation-thread] Started rotation thread");
 936
 937         if (!handle) {
 938                 ERR("[rotation-thread] Invalid thread context provided");
 939                 goto end;
 940         }
 941
 942         rcu_register_thread();
 943         rcu_thread_online();
 944
 945         health_register(health_sessiond, HEALTH_SESSIOND_TYPE_ROTATION);
 946         health_code_update();
 947
 948         ret = init_thread_state(handle, &thread);
 949         if (ret) {
 950                 goto error;
 951         }
 952
 953         while (true) {
 954                 int fd_count, i;
 955
 956                 health_poll_entry();
 957                 DBG("[rotation-thread] Entering poll wait");
 958                 ret = lttng_poll_wait(&thread.events, -1);
 959                 DBG("[rotation-thread] Poll wait returned (%i)", ret);
 960                 health_poll_exit();
 961                 if (ret < 0) {
 962                         /*
 963                          * Restart interrupted system call.
 964                          */
 965                         if (errno == EINTR) {
 966                                 continue;
 967                         }
 968                         ERR("[rotation-thread] Error encountered during lttng_poll_wait (%i)", ret);
 969                         goto error;
 970                 }
 971
 972                 fd_count = ret;
 973                 for (i = 0; i < fd_count; i++) {
 974                         int fd = LTTNG_POLL_GETFD(&thread.events, i);
 975                         uint32_t revents = LTTNG_POLL_GETEV(&thread.events, i);
 976
 977                         DBG("[rotation-thread] Handling fd (%i) activity (%u)",
 978                                         fd, revents);
 979
 980                         if (revents & LPOLLERR) {
 981                                 ERR("[rotation-thread] Polling returned an error on fd %i", fd);
 982                                 goto error;
 983                         }
 984
 985                         if (fd == rotate_notification_channel->socket) {
 986                                 ret = handle_notification_channel(fd, handle,
 987                                                 &thread);
 988                                 if (ret) {
 989                                         ERR("[rotation-thread] Error occurred while handling activity on notification channel socket");
 990                                         goto error;
 991                                 }
 992                         } else {
 993                                 /* Job queue or quit pipe activity. */
 994
 995                                 /*
 996                                  * The job queue is serviced if there is
 997                                  * activity on the quit pipe to ensure it is
 998                                  * flushed and all references held in the queue
 999                                  * are released.
1000                                  */
1001                                 ret = handle_job_queue(handle, &thread,
1002                                                 handle->rotation_timer_queue);
1003                                 if (ret) {
1004                                         ERR("[rotation-thread] Failed to handle rotation timer pipe event");
1005                                         goto error;
1006                                 }
1007
1008                                 if (fd == queue_pipe_fd) {
1009                                         char buf;
1010
1011                                         ret = lttng_read(fd, &buf, 1);
1012                                         if (ret != 1) {
1013                                                 ERR("[rotation-thread] Failed to read from wakeup pipe (fd = %i)", fd);
1014                                                 ret = -1;
1015                                                 goto error;
1016                                         }
1017                                 } else {
1018                                         DBG("[rotation-thread] Quit pipe activity");
1019                                         goto exit;
1020                                 }
1021                         }
1022                 }
1023         }
1024 exit:
1025 error:
1026         DBG("[rotation-thread] Exit");
1027         fini_thread_state(&thread);
1028         health_unregister(health_sessiond);
1029         rcu_thread_offline();
1030         rcu_unregister_thread();
1031 end:
1032         return NULL;
1033 }
1034
1035 static
1036 bool shutdown_rotation_thread(void *thread_data)
1037 {
1038         struct rotation_thread_handle *handle = thread_data;
1039         const int write_fd = lttng_pipe_get_writefd(handle->quit_pipe);
1040
1041         return notify_thread_pipe(write_fd) == 1;
1042 }
1043
1044 bool launch_rotation_thread(struct rotation_thread_handle *handle)
1045 {
1046         struct lttng_thread *thread;
1047
1048         thread = lttng_thread_create("Rotation",
1049                         thread_rotation,
1050                         shutdown_rotation_thread,
1051                         NULL,
1052                         handle);
1053         if (!thread) {
1054                 goto error;
1055         }
1056         lttng_thread_put(thread);
1057         return true;
1058 error:
1059         return false;
1060 }