From 2f7279c16935811ad86fc0fcaa95d55614e724d4 Mon Sep 17 00:00:00 2001 From: Kienan Stewart Date: Mon, 8 Jan 2024 15:29:20 -0500 Subject: [PATCH] ansible: Install rasdaemon and prometheus exporter on CI hosts Change-Id: Iff4018e7e4174aad321ab740e534cab4d3414255 Signed-off-by: Kienan Stewart --- automation/ansible/hosts.yml | 1 + .../ansible/roles/rasdaemon/defaults/main.yml | 6 ++ .../rasdaemon/files/rasdaemon-exporter.py | 94 +++++++++++++++++++ .../ansible/roles/rasdaemon/handlers/main.yml | 8 ++ .../ansible/roles/rasdaemon/tasks/main.yml | 10 ++ .../roles/rasdaemon/tasks/prometheus.yml | 25 +++++ .../templates/rasdaemon-exporter.service.j2 | 10 ++ 7 files changed, 154 insertions(+) create mode 100644 automation/ansible/roles/rasdaemon/defaults/main.yml create mode 100755 automation/ansible/roles/rasdaemon/files/rasdaemon-exporter.py create mode 100644 automation/ansible/roles/rasdaemon/handlers/main.yml create mode 100644 automation/ansible/roles/rasdaemon/tasks/main.yml create mode 100644 automation/ansible/roles/rasdaemon/tasks/prometheus.yml create mode 100644 automation/ansible/roles/rasdaemon/templates/rasdaemon-exporter.service.j2 diff --git a/automation/ansible/hosts.yml b/automation/ansible/hosts.yml index 9fc739f..33b9062 100644 --- a/automation/ansible/hosts.yml +++ b/automation/ansible/hosts.yml @@ -7,3 +7,4 @@ - common - libvirt - lxd + - rasdaemon diff --git a/automation/ansible/roles/rasdaemon/defaults/main.yml b/automation/ansible/roles/rasdaemon/defaults/main.yml new file mode 100644 index 0000000..11ad6f1 --- /dev/null +++ b/automation/ansible/roles/rasdaemon/defaults/main.yml @@ -0,0 +1,6 @@ +--- +rasdaemon_prometheus_exporter: true +rasdaemon_prometheus_exporter_bind_address: '0.0.0.0' +rasdaemon_prometheus_exporter_port: 9797 +rasdaemon_prometheus_exporter_prerequisites: + - python3 diff --git a/automation/ansible/roles/rasdaemon/files/rasdaemon-exporter.py b/automation/ansible/roles/rasdaemon/files/rasdaemon-exporter.py new file mode 100755 index 0000000..7613957 --- /dev/null +++ b/automation/ansible/roles/rasdaemon/files/rasdaemon-exporter.py @@ -0,0 +1,94 @@ +#!/usr/bin/python3 +# +# SPDX-FileCopyrightText: 2024 Kienan Stewart +# SPDX-License-Identifier: GPL-2.0-or-later +# +# Prometheus exporter for rasdaemon +# +# Based on https://github.com/openstreetmap/prometheus-exporters/blob/main/exporters/rasdaemon/rasdaemon_exporter +# + +import argparse +import http.server +import sqlite3 +import urllib.parse + +METRICS = { + 'rasdaemon_mc_events_total': { + 'help': 'Memory controller errors', + 'type': 'counter', + 'query': 'SELECT mc, top_layer, middle_layer, lower_layer, err_type, SUM(err_count) as err_count FROM mc_event GROUP BY mc, top_layer, middle_layer, lower_layer, err_type', + }, +} + +class ExporterServer(http.server.ThreadingHTTPServer): + + def __init__(self, server_address, RequestHandlerClass, db_file='/var/lib/rasdaemon/ras-mc_event.db'): + super().__init__(server_address, RequestHandlerClass) + self.db_file = db_file + + +class ExporterHandler(http.server.BaseHTTPRequestHandler): + + def do_GET(self): + url = urllib.parse.urlparse(self.path) + if url.path != '/metrics': + self.send_response(404) + self.end_headers() + return + + metrics_details = {key: dict() for (key, _) in METRICS.items()} + with sqlite3.connect('file:{}?mode=ro'.format(self.server.db_file), uri=True) as con: + cursor = con.cursor() + cursor.row_factory = sqlite3.Row + for key, _ in METRICS.items(): + result = cursor.execute(METRICS[key]['query']) + metrics_details[key] = result.fetchall() + + print(metrics_details) + self.send_response(200) + self.end_headers() + for key, value in METRICS.items(): + self.wfile.write("# HELP {} {}\n".format(key, value['help']).encode()) + self.wfile.write("# TYPE {} {}\n".format(key, value['type']).encode()) + if not metrics_details[key]: + self.wfile.write("{} 0\n".format(key).encode()) + for entry in metrics_details[key]: + labels = ','.join(['{}="{}"'.format(key, entry[key]) for key in entry.keys() if key != 'err_count']) + if labels: + self.wfile.write("{}{{{}}} {}\n".format(key, labels, entry['err_count']).encode()) + else: + self.wfile.write("{} {}\n".format(key, entry['err_count']).encode()) + + +def _get_argument_parser(): + parser = argparse.ArgumentParser( + prog='rasdaemon-exporter', + description='Exporters rasdaemon metrics' + ) + parser.add_argument( + '-p', '--port', type=int, default=9797, + help='The port to listen on' + ) + parser.add_argument( + '-l', '--listen-address', type=str, default='', + help='The address to listen on' + ) + parser.add_argument( + '-f', '--rasdaemon-db-file', type=str, #type=argparse.FileType('r'), + default='/var/lib/rasdaemon/ras-mc_event.db', + help='The path to the rasdaemon sqlite3 database' + ) + return parser + + +def serve(listen_address, listen_port, db_file): + with ExporterServer((listen_address, listen_port), ExporterHandler, db_file) as httpd: + httpd.serve_forever() + print('done') + + +if __name__ == '__main__': + parser = _get_argument_parser() + args = parser.parse_args() + serve(args.listen_address, args.port, args.rasdaemon_db_file) diff --git a/automation/ansible/roles/rasdaemon/handlers/main.yml b/automation/ansible/roles/rasdaemon/handlers/main.yml new file mode 100644 index 0000000..a4f487e --- /dev/null +++ b/automation/ansible/roles/rasdaemon/handlers/main.yml @@ -0,0 +1,8 @@ +--- +- name: Systemd daemon-reload + ansible.builtin.systemd: + daemon_reload: true +- name: Restart rasdaemon-exporter + ansible.builtin.service: + name: rasdaemon-exporter + state: restarted diff --git a/automation/ansible/roles/rasdaemon/tasks/main.yml b/automation/ansible/roles/rasdaemon/tasks/main.yml new file mode 100644 index 0000000..c2c8657 --- /dev/null +++ b/automation/ansible/roles/rasdaemon/tasks/main.yml @@ -0,0 +1,10 @@ +--- +- name: Install rasdaemon + ansible.builtin.package: + name: rasdaemon +- name: Ensure rasdaemon is running + ansible.builtin.service: + name: rasdaemon + state: started + enabled: true +- ansible.builtin.include: prometheus.yml diff --git a/automation/ansible/roles/rasdaemon/tasks/prometheus.yml b/automation/ansible/roles/rasdaemon/tasks/prometheus.yml new file mode 100644 index 0000000..03b6213 --- /dev/null +++ b/automation/ansible/roles/rasdaemon/tasks/prometheus.yml @@ -0,0 +1,25 @@ +--- +- name: Install rasdaemon prometheus exporter requirements + ansible.builtin.package: + name: "{{rasdaemon_prometheus_exporter_prerequisites}}" +- name: Install rasdaemon prometheus exporter + ansible.builtin.copy: + dest: '/usr/local/bin/rasdaemon-exporter' + src: 'rasdaemon-exporter.py' + owner: 'root' + group: 'root' + mode: '0755' + notify: + - Restart rasdaemon-exporter +- name: Deploy rasdaemon prometheus exporter service + ansible.builtin.template: + src: 'rasdaemon-exporter.service.j2' + dest: '/etc/systemd/system/rasdaemon-exporter.service' + notify: + - Systemd daemon-reload + - Restart rasdaemon-exporter +- name: Ensure rasdaemon prometheus exporter service is running + ansible.builtin.service: + name: rasdaemon-exporter + enabled: "{{rasdaemon_prometheus_exporter}}" + state: "{{rasdaemon_prometheus_exporter|ternary('started', 'stopped')}}" diff --git a/automation/ansible/roles/rasdaemon/templates/rasdaemon-exporter.service.j2 b/automation/ansible/roles/rasdaemon/templates/rasdaemon-exporter.service.j2 new file mode 100644 index 0000000..4e2b859 --- /dev/null +++ b/automation/ansible/roles/rasdaemon/templates/rasdaemon-exporter.service.j2 @@ -0,0 +1,10 @@ +[Unit] +Description=Rasdaemon Prometheus Exporter +After=network.target + +[Service] +Type=simple +ExecStart=/usr/local/bin/rasdaemon-exporter -l "{{rasdaemon_prometheus_exporter_bind_address}}" -p "{{rasdaemon_prometheus_exporter_port}}" + +[Install] +WantedBy=multi-user.target -- 2.34.1