* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/
+#define _LGPL_SOURCE
#include <sys/types.h>
#include <unistd.h>
#include <string.h>
#include <stdlib.h>
#include <stdio.h>
+#include <stdbool.h>
#include <sys/mman.h>
#include <sys/syscall.h>
-#include <linux/perf_event.h>
#include <lttng/ust-events.h>
#include <lttng/ust-tracer.h>
#include <lttng/ringbuffer-config.h>
#include <urcu/ref.h>
#include <usterr-signal-safe.h>
#include <signal.h>
+#include "perf_event.h"
#include "lttng-tracer-core.h"
/*
return size;
}
+static
+uint64_t read_perf_counter_syscall(
+ struct lttng_perf_counter_thread_field *thread_field)
+{
+ uint64_t count;
+
+ if (caa_unlikely(thread_field->fd < 0))
+ return 0;
+
+ if (caa_unlikely(read(thread_field->fd, &count, sizeof(count))
+ < sizeof(count)))
+ return 0;
+
+ return count;
+}
+
#if defined(__x86_64__) || defined(__i386__)
static
return low | ((uint64_t) high) << 32;
}
-static bool arch_perf_use_read(void)
+static
+bool has_rdpmc(struct perf_event_mmap_page *pc)
{
- return false;
+ if (caa_unlikely(!pc->cap_bit0_is_deprecated))
+ return false;
+ /* Since Linux kernel 3.12. */
+ return pc->cap_user_rdpmc;
}
-#else /* defined(__x86_64__) || defined(__i386__) */
-
-#error "Perf event counters are only supported on x86 so far."
-
-#endif /* #else defined(__x86_64__) || defined(__i386__) */
-
static
-uint64_t read_perf_counter(struct perf_event_mmap_page *pc)
+uint64_t arch_read_perf_counter(
+ struct lttng_perf_counter_thread_field *thread_field)
{
uint32_t seq, idx;
uint64_t count;
+ struct perf_event_mmap_page *pc = thread_field->pc;
if (caa_unlikely(!pc))
return 0;
cmm_barrier();
idx = pc->index;
- if (idx)
- count = pc->offset + rdpmc(idx - 1);
- else
- count = 0;
-
+ if (caa_likely(has_rdpmc(pc) && idx)) {
+ int64_t pmcval;
+
+ pmcval = rdpmc(idx - 1);
+ /* Sign-extend the pmc register result. */
+ pmcval <<= 64 - pc->pmc_width;
+ pmcval >>= 64 - pc->pmc_width;
+ count = pc->offset + pmcval;
+ } else {
+ /* Fall-back on system call if rdpmc cannot be used. */
+ return read_perf_counter_syscall(thread_field);
+ }
cmm_barrier();
} while (CMM_LOAD_SHARED(pc->lock) != seq);
return count;
}
+static
+int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
+{
+ struct perf_event_mmap_page *pc = thread_field->pc;
+
+ if (!pc)
+ return 0;
+ return !has_rdpmc(pc);
+}
+
+#else
+
+/* Generic (slow) implementation using a read system call. */
+static
+uint64_t arch_read_perf_counter(
+ struct lttng_perf_counter_thread_field *thread_field)
+{
+ return read_perf_counter_syscall(thread_field);
+}
+
+static
+int arch_perf_keep_fd(struct lttng_perf_counter_thread_field *thread_field)
+{
+ return 1;
+}
+
+#endif
+
static
int sys_perf_event_open(struct perf_event_attr *attr,
pid_t pid, int cpu, int group_fd,
}
static
-struct perf_event_mmap_page *setup_perf(
- struct lttng_perf_counter_thread_field *thread_field)
+void close_perf_fd(int fd)
+{
+ int ret;
+
+ if (fd < 0)
+ return;
+
+ ret = close(fd);
+ if (ret) {
+ perror("Error closing LTTng-UST perf memory mapping FD");
+ }
+}
+
+static void setup_perf(struct lttng_perf_counter_thread_field *thread_field)
{
void *perf_addr;
PROT_READ, MAP_SHARED, thread_field->fd, 0);
if (perf_addr == MAP_FAILED)
perf_addr = NULL;
+ thread_field->pc = perf_addr;
- if (!arch_perf_use_read()) {
+ if (!arch_perf_keep_fd(thread_field)) {
close_perf_fd(thread_field->fd);
thread_field->fd = -1;
}
-
-end:
- return perf_addr;
-}
-
-static
-void close_perf_fd(int fd)
-{
- int ret;
-
- if (fd < 0)
- return;
-
- ret = close(fd);
- if (ret)
- perror("Error closing LTTng-UST perf memory mapping FD");
}
static
thread_field->field = perf_field;
thread_field->fd = open_perf_fd(&perf_field->attr);
if (thread_field->fd >= 0)
- thread_field->pc = setup_perf(thread_field);
+ setup_perf(thread_field);
/*
* Note: thread_field->pc can be NULL if setup_perf() fails.
* Also, thread_field->fd can be -1 if open_perf_fd() fails.
perf_field = field->u.perf_counter;
perf_thread_field = get_thread_field(perf_field);
- return read_perf_counter(perf_thread_field->pc);
+ return arch_read_perf_counter(perf_thread_field);
}
static
free(perf_field);
}
+#ifdef __ARM_ARCH_7A__
+
+static
+int perf_get_exclude_kernel(void)
+{
+ return 0;
+}
+
+#else /* __ARM_ARCH_7A__ */
+
+static
+int perf_get_exclude_kernel(void)
+{
+ return 1;
+}
+
+#endif /* __ARM_ARCH_7A__ */
+
/* Called with UST lock held */
int lttng_add_perf_counter_to_ctx(uint32_t type,
uint64_t config,
perf_field->attr.type = type;
perf_field->attr.config = config;
- perf_field->attr.exclude_kernel = 1;
+ perf_field->attr.exclude_kernel = perf_get_exclude_kernel();
CDS_INIT_LIST_HEAD(&perf_field->thread_field_list);
field->u.perf_counter = perf_field;