From: Kienan Stewart Date: Thu, 18 Jan 2024 19:23:20 +0000 (-0500) Subject: ansible: Install gcc5 on Debian 12 CI nodes X-Git-Url: http://git.liburcu.org/?a=commitdiff_plain;h=4f6c1ba91db07c3c42f7f294f4e75ee61227d48a;p=lttng-ci.git ansible: Install gcc5 on Debian 12 CI nodes lttng-modules crossbuilds for arm64 were failing for kernel versions < 4.4, principally due to gcc < 5.1 being explicitly denied in lttng-modules. To recover compilation coverage for these versions on this architecture, gcc 5.5.0 has been built and deployed via ansible on to the deb12-amd64 nodes. The compilation coverage is not quite as large, as support for gcc 5 was only introduced in linux 3.18. Fixes: support #1604 Change-Id: I49e7fc9d081c0b86cb25487d161efc3d1cf8f484 Signed-off-by: Kienan Stewart --- diff --git a/automation/ansible/roles/cross-compilers/files/Makefile b/automation/ansible/roles/cross-compilers/files/Makefile new file mode 100644 index 0000000..58187f4 --- /dev/null +++ b/automation/ansible/roles/cross-compilers/files/Makefile @@ -0,0 +1,28 @@ +.PHONY: gcc48 gcc48-container gcc55 gcc55-container + +all: gcc48 gcc55 + +#ARCHES = x86_64-pc-linux-gnu aarch64-linux-gnu arm-linux-gnueabihf i686-linux-gnu powerpc64le-linux-gnu powerpc-linux-gnu riscv64-linux-gnu s390x-linux-gnu +ARCHES = x86_64-pc-linux-gnu aarch64-linux-gnu +# The last build of gcc-4.8 used a native gcc-4.8 compiler from snapshot.debian.org, +# so we avoid rebuilding the host compiler at this time. +ARCHES_48 := $(filter-out x86_64-pc-linux-gnu,$(ARCHES)) + +gcc55: gcc55-container + mkdir -p gcc55 + podman run --rm -e "TARGETS=$(ARCHES)" -e "SRC_DIR=/src/gcc-releases-gcc-5.5.0" -e "BIN_SUFFIX=5.5" -e "CSTD=gnu11" -e "CXXSTD=gnu++11" -v ./gcc55:/output localhost/gcc-5.5 + tar -czf gcc55.tar.gz -C gcc55 ./ + +gcc55-container: + podman build -t gcc-5.5 -f gcc5.Containerfile . + +gcc48: gcc48-container + mkdir -p gcc48 + podman run --rm -e "TARGETS=$(ARCHES_48)" -v ./gcc48:/output localhost/gcc-4.8 + tar -czf gcc48.tar.gz -C gcc48 ./ + +gcc48-container: + podman build -t gcc-4.8 -f Containerfile . + +clean: + rm -rf gcc55/ gcc48/ diff --git a/automation/ansible/roles/cross-compilers/files/gcc5-cyclades_removal.patch b/automation/ansible/roles/cross-compilers/files/gcc5-cyclades_removal.patch new file mode 100644 index 0000000..1ecf605 --- /dev/null +++ b/automation/ansible/roles/cross-compilers/files/gcc5-cyclades_removal.patch @@ -0,0 +1,123 @@ +# https://gcc.gnu.org/git/?p=gcc.git;a=patch;h=2b40941d23b1570cdd90083b58fa0f66aa58c86e +# Modified to apply on gcc 5.5.0 +# +From 2b40941d23b1570cdd90083b58fa0f66aa58c86e Mon Sep 17 00:00:00 2001 +From: Tamar Christina +Date: Fri, 21 May 2021 12:16:56 +0100 +Subject: [PATCH] libsanitizer: Remove cyclades from libsanitizer + +The Linux kernel has removed the interface to cyclades from +the latest kernel headers[1] due to them being orphaned for the +past 13 years. + +libsanitizer uses this header when compiling against glibc, but +glibcs itself doesn't seem to have any references to cyclades. + +Further more it seems that the driver is broken in the kernel and +the firmware doesn't seem to be available anymore. + +As such since this is breaking the build of libsanitizer (and so the +GCC bootstrap[2]) I propose to remove this. + +[1] https://lkml.org/lkml/2021/3/2/153 +[2] https://gcc.gnu.org/bugzilla/show_bug.cgi?id=100379 + +libsanitizer/ChangeLog: + + PR sanitizer/100379 + * sanitizer_common/sanitizer_common_interceptors_ioctl.inc: Cherry-pick + llvm-project revision f7c5351552387bd43f6ca3631016d7f0dfe0f135. + * sanitizer_common/sanitizer_platform_limits_posix.cc: Likewise. + * sanitizer_common/sanitizer_platform_limits_posix.h: Likewise. +--- + .../sanitizer_common_interceptors_ioctl.inc | 9 --------- + .../sanitizer_platform_limits_posix.cc | 11 ----------- + .../sanitizer_platform_limits_posix.h | 10 ---------- + 3 files changed, 30 deletions(-) + +diff --git a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_ioctl.inc b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +index 5408ea17c59..7a9cd3f5968 100644 +--- a/libsanitizer/sanitizer_common/sanitizer_common_interceptors_ioctl.inc ++++ b/libsanitizer/sanitizer_common/sanitizer_common_interceptors_ioctl.inc +@@ -365,15 +365,6 @@ static void ioctl_table_fill() { + + #if SANITIZER_LINUX && !SANITIZER_ANDROID + // _(SIOCDEVPLIP, WRITE, struct_ifreq_sz); // the same as EQL_ENSLAVE +- _(CYGETDEFTHRESH, WRITE, sizeof(int)); +- _(CYGETDEFTIMEOUT, WRITE, sizeof(int)); +- _(CYGETMON, WRITE, struct_cyclades_monitor_sz); +- _(CYGETTHRESH, WRITE, sizeof(int)); +- _(CYGETTIMEOUT, WRITE, sizeof(int)); +- _(CYSETDEFTHRESH, NONE, 0); +- _(CYSETDEFTIMEOUT, NONE, 0); +- _(CYSETTHRESH, NONE, 0); +- _(CYSETTIMEOUT, NONE, 0); + _(EQL_EMANCIPATE, WRITE, struct_ifreq_sz); + _(EQL_ENSLAVE, WRITE, struct_ifreq_sz); + _(EQL_GETMASTRCFG, WRITE, struct_ifreq_sz); +diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc +index d823a12190c..e8fce8a0287 100644 +--- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc ++++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc +@@ -157,6 +157,5 @@ typedef struct user_fpregs elf_fpregset_t; + #include + #include +-#include + #include + #include + #include +@@ -466,7 +465,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); + + #if SANITIZER_LINUX && !SANITIZER_ANDROID + unsigned struct_ax25_parms_struct_sz = sizeof(struct ax25_parms_struct); +- unsigned struct_cyclades_monitor_sz = sizeof(struct cyclades_monitor); + #if EV_VERSION > (0x010000) + unsigned struct_input_keymap_entry_sz = sizeof(struct input_keymap_entry); + #else +@@ -833,15 +831,6 @@ unsigned struct_ElfW_Phdr_sz = sizeof(Elf_Phdr); + #endif // SANITIZER_LINUX || SANITIZER_FREEBSD + + #if SANITIZER_LINUX && !SANITIZER_ANDROID +- unsigned IOCTL_CYGETDEFTHRESH = CYGETDEFTHRESH; +- unsigned IOCTL_CYGETDEFTIMEOUT = CYGETDEFTIMEOUT; +- unsigned IOCTL_CYGETMON = CYGETMON; +- unsigned IOCTL_CYGETTHRESH = CYGETTHRESH; +- unsigned IOCTL_CYGETTIMEOUT = CYGETTIMEOUT; +- unsigned IOCTL_CYSETDEFTHRESH = CYSETDEFTHRESH; +- unsigned IOCTL_CYSETDEFTIMEOUT = CYSETDEFTIMEOUT; +- unsigned IOCTL_CYSETTHRESH = CYSETTHRESH; +- unsigned IOCTL_CYSETTIMEOUT = CYSETTIMEOUT; + unsigned IOCTL_EQL_EMANCIPATE = EQL_EMANCIPATE; + unsigned IOCTL_EQL_ENSLAVE = EQL_ENSLAVE; + unsigned IOCTL_EQL_GETMASTRCFG = EQL_GETMASTRCFG; +diff --git a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h +index 6a673a7c995..f921bf2b5b5 100644 +--- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h ++++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.h +@@ -1040,7 +1040,6 @@ struct __sanitizer_cookie_io_functions_t { + + #if SANITIZER_LINUX && !SANITIZER_ANDROID + extern unsigned struct_ax25_parms_struct_sz; +- extern unsigned struct_cyclades_monitor_sz; + extern unsigned struct_input_keymap_entry_sz; + extern unsigned struct_ipx_config_data_sz; + extern unsigned struct_kbdiacrs_sz; +@@ -1385,15 +1384,6 @@ struct __sanitizer_cookie_io_functions_t { + #endif // SANITIZER_LINUX || SANITIZER_FREEBSD + + #if SANITIZER_LINUX && !SANITIZER_ANDROID +- extern unsigned IOCTL_CYGETDEFTHRESH; +- extern unsigned IOCTL_CYGETDEFTIMEOUT; +- extern unsigned IOCTL_CYGETMON; +- extern unsigned IOCTL_CYGETTHRESH; +- extern unsigned IOCTL_CYGETTIMEOUT; +- extern unsigned IOCTL_CYSETDEFTHRESH; +- extern unsigned IOCTL_CYSETDEFTIMEOUT; +- extern unsigned IOCTL_CYSETTHRESH; +- extern unsigned IOCTL_CYSETTIMEOUT; + extern unsigned IOCTL_EQL_EMANCIPATE; + extern unsigned IOCTL_EQL_ENSLAVE; + extern unsigned IOCTL_EQL_GETMASTRCFG; +-- +2.39.3 + diff --git a/automation/ansible/roles/cross-compilers/files/gcc5-isl.patch b/automation/ansible/roles/cross-compilers/files/gcc5-isl.patch new file mode 100644 index 0000000..683a48a --- /dev/null +++ b/automation/ansible/roles/cross-compilers/files/gcc5-isl.patch @@ -0,0 +1,59 @@ +From 114e4e9470260a839d55aad2421fb646af12697b Mon Sep 17 00:00:00 2001 +From: Paul Beeler +Date: Mon, 11 May 2015 22:27:25 -0600 +Subject: [PATCH] Graphite: Add missing isl headers + +Signed-off-by: Paul Beeler +# Patch modified from original source +# https://github.com/SaberMod/GCC_SaberMod/commit/114e4e9470260a839d55aad2421fb646af12697b.patch +--- + gcc/graphite-isl-ast-to-gimple.c | 1 + + gcc/graphite-optimize-isl.c | 1 + + gcc/graphite-poly.h | 6 ++++++ + 3 files changed, 8 insertions(+), 2 deletions(-) + +--- a/gcc/graphite-isl-ast-to-gimple.c ++++ b/gcc/graphite-isl-ast-to-gimple.c +@@ -21,9 +21,10 @@ along with GCC; see the file COPYING3. If not see + #include "config.h" + + #ifdef HAVE_isl ++#include + #include + #include + #include + #include + #include + #include + + /* Since ISL-0.13, the extern is in val_gmp.h. */ +diff --git a/gcc/graphite-optimize-isl.c b/gcc/graphite-optimize-isl.c +index 8bdf7443018..d602e672ca8 100644 +--- a/gcc/graphite-optimize-isl.c ++++ b/gcc/graphite-optimize-isl.c +@@ -28,6 +28,7 @@ along with GCC; see the file COPYING3. If not see + #include + #include + #include ++#include + #endif + + #include "system.h" +diff --git a/gcc/graphite-poly.h b/gcc/graphite-poly.h +index afc0121f611..06d91faa1ba 100644 +--- a/gcc/graphite-poly.h ++++ b/gcc/graphite-poly.h +@@ -22,6 +22,13 @@ along with GCC; see the file COPYING3. If not see + #ifndef GCC_GRAPHITE_POLY_H + #define GCC_GRAPHITE_POLY_H + ++#ifdef HAVE_isl ++#include ++#include ++#include ++#include ++#endif ++ + typedef struct poly_dr *poly_dr_p; + + typedef struct poly_bb *poly_bb_p; diff --git a/automation/ansible/roles/cross-compilers/files/gcc5-sanitizer_fs.patch b/automation/ansible/roles/cross-compilers/files/gcc5-sanitizer_fs.patch new file mode 100644 index 0000000..f1d11b9 --- /dev/null +++ b/automation/ansible/roles/cross-compilers/files/gcc5-sanitizer_fs.patch @@ -0,0 +1,25 @@ +--- a/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc 2024-01-17 21:18:45.979243850 +0000 ++++ b/libsanitizer/sanitizer_common/sanitizer_platform_limits_posix.cc 2024-01-17 21:20:13.467971880 +0000 +@@ -60,7 +60,6 @@ + #include + #include + #include +-#include + #include + #include + #include +@@ -767,10 +766,10 @@ + unsigned IOCTL_EVIOCGPROP = IOCTL_NOT_PRESENT; + unsigned IOCTL_EVIOCSKEYCODE_V2 = IOCTL_NOT_PRESENT; + #endif +- unsigned IOCTL_FS_IOC_GETFLAGS = FS_IOC_GETFLAGS; +- unsigned IOCTL_FS_IOC_GETVERSION = FS_IOC_GETVERSION; +- unsigned IOCTL_FS_IOC_SETFLAGS = FS_IOC_SETFLAGS; +- unsigned IOCTL_FS_IOC_SETVERSION = FS_IOC_SETVERSION; ++ unsigned IOCTL_FS_IOC_GETFLAGS = _IOR('f', 1, long); ++ unsigned IOCTL_FS_IOC_GETVERSION = _IOR('v', 1, long); ++ unsigned IOCTL_FS_IOC_SETFLAGS = _IOW('f', 2, long); ++ unsigned IOCTL_FS_IOC_SETVERSION = _IOW('v', 2, long); + unsigned IOCTL_GIO_CMAP = GIO_CMAP; + unsigned IOCTL_GIO_FONT = GIO_FONT; + unsigned IOCTL_GIO_UNIMAP = GIO_UNIMAP; \ No newline at end of file diff --git a/automation/ansible/roles/cross-compilers/files/gcc5.Containerfile b/automation/ansible/roles/cross-compilers/files/gcc5.Containerfile new file mode 100644 index 0000000..a06e93a --- /dev/null +++ b/automation/ansible/roles/cross-compilers/files/gcc5.Containerfile @@ -0,0 +1,51 @@ +FROM docker.io/debian:bookworm + +# +# Create the container: podman build -t gcc-5 -f gcc5.Containerfile . +# Build the cross compilers: +# mkdir -p ~/gcc-5 +# for i in aarch64-linux-gnu arm-linux-gnueabihf i686-linux-gnu powerpc64le-linux-gnu powerpc-linux-gnu riscv64-linux-gnu s390x-linux-gnu ; do podman run --rm -e "TARGET=$i" -e "SRC_DIR=/src/gcc-releases-gcc-5.5.0" -e "BIN_SUFFIX=5" -e "CSTD=gnu11" -e "CXXSTD=gnu++11" -v ~/gcc-5:/output localhost/gcc-5 ; done +# tar -czf ~/gcc-5.tgz -C ~/gcc-5 ./ + +RUN echo 'deb-src http://deb.debian.org/debian bookworm main contrib' >> /etc/apt/sources.list +RUN apt-get update + +RUN apt-get -y --force-yes build-dep gcc-12 +RUN apt-get -y --force-yes install wget +RUN apt-get -y --force-yes install gcc-12-aarch64-linux-gnu gcc-12-riscv64-linux-gnu gcc-12-i686-linux-gnu gcc-12-s390x-linux-gnu gcc-12-powerpc64le-linux-gnu gcc-12-powerpc-linux-gnu gcc-12-arm-linux-gnueabihf +# gcc-5.5.0 isn't compatible with libisl >= 0.21, as that version drops +# the deprecated isl_map_n_out function. +# See libisl commit aed01a35f1cd18ec6c9d3d151aee8d66afd67610 +# gcc-5.5.0 ins't compatible with libisl >= 0.19 as isl_band.h was removed in 0.19. +# See libisl commit 4eb5d8f7ea10bd5b47022334640f3c074f5a3d40 +# +# RUN wget -q http://snapshot.debian.org/archive/debian/20170106T032927Z/pool/main/i/isl/libisl-dev_0.18-1_amd64.deb +# RUN wget -q http://snapshot.debian.org/archive/debian/20170106T032927Z/pool/main/i/isl/libisl15_0.18-1_amd64.deb +# RUN apt install -y --allow-downgrades ./*.deb + +RUN mkdir -p /src/build /src/patches /output +WORKDIR /src +RUN wget -q -O - https://github.com/gcc-mirror/gcc/archive/refs/tags/releases/gcc-5.5.0.tar.gz | tar -xzf - + +WORKDIR /src/patches +# ustat.h was removed from glibc 2.28; the build of the compiler piggy backs on +# glibc shipped with the base OS, so a patch to address it's removal is +# necessary to allow the build to pass. +RUN wget -q https://aur.archlinux.org/cgit/aur.git/plain/glibc2.28-ustat.patch?h=gcc5 -O 01-ustat_removal.patch.0 + +# recent linux kernels removed support for cyclades +COPY gcc5-cyclades_removal.patch /src/patches/02-cyclades_removal.patch.1 + +# backport of https://github.com/gcc-mirror/gcc/commit/2701442d0cf6292f6624443c15813d6d1a3562fe +COPY gcc5-sanitizer_fs.patch /src/patches/03-sanitizer_fs.patch.1 + +WORKDIR /src/build +COPY script.sh /usr/bin/build-gcc.sh +CMD /usr/bin/build-gcc.sh + +# @TODO: missing crtbegin.o eg., libgcc-5-dev-arm64-cross +# This can be "worked around" by copying the system libc-cross +# eg. +# cp -r /usr/lib/gcc-cross/aarch64-linux-gnu/12/ /usr/lib/gcc-cross/aarch64-linux-gnu/5.5.0 +# make +# diff --git a/automation/ansible/roles/cross-compilers/files/script.sh b/automation/ansible/roles/cross-compilers/files/script.sh index e6fec0b..b868787 100755 --- a/automation/ansible/roles/cross-compilers/files/script.sh +++ b/automation/ansible/roles/cross-compilers/files/script.sh @@ -1,15 +1,23 @@ #!/bin/bash -x +# +# SPDX-FileCopyrightText: 2024 Kienan Stewart "/output/${TARGET}.time" + fi -mkdir -p "/output/usr/lib/gcc-cross/${TARGET}" /output/usr/bin/ + cp config.log "/output/config-${TARGET}.log" +done -if [ -n "${DEBUG}" ] ; then - echo $(($(date +%s) - START)) > "/output/${TARGET}.time" -fi +cp -r "/usr/local/gcc${BIN_SUFFIX}" "/output/usr/local/" -cp -r /usr/lib/gcc-cross/"${TARGET}"/4* "/output/usr/lib/gcc-cross/${TARGET}/" -cp /usr/bin/*-4.8 /output/usr/bin/ +# To test +# 1. Copy the output tarball from the Makefile to a Debian bookworm instance +# 2. Unpack, eg. `tar -xzf gcc55.tar.gz -C /` +# 3. Build a small test program, eg. +# LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/gcc5.5/bin \ +# PATH=$PATH:/usr/local/gcc5.5/bin \ +# aarch64-linux-gnu-gcc-5.5 -o hello hello.c +# 4. Copy the built binary to a system with that native architecture and run it +# 5. Upload to object storage, eg. +# s3cmd put gcc55.tar.gz s3://jenkins/gcc-5.5-x86_64-linux-gnu-cross.tgz +# diff --git a/automation/ansible/roles/cross-compilers/tasks/setup-Debian.yml b/automation/ansible/roles/cross-compilers/tasks/setup-Debian.yml index a03cfa9..bb7152e 100644 --- a/automation/ansible/roles/cross-compilers/tasks/setup-Debian.yml +++ b/automation/ansible/roles/cross-compilers/tasks/setup-Debian.yml @@ -35,13 +35,23 @@ - powerpc-linux-gnu - riscv64-linux-gnu - s390x-linux-gnu - - name: Download legacy compiler binaries + - name: Download gcc-4.8 cross compiler binaries # These binaries built from files/Containerfile ansible.builtin.get_url: url: https://obj.internal.efficios.com/jenkins/gcc-4.8-x86_64-linux-gnu-cross.tgz dest: /root/gcc-4.8-x86_64-linux-gnu-cross.tgz - register: cross_download - - name: Extract legacy compiler binaries - when: cross_download.changed + register: cross_gcc48_download + - name: Extract gcc-4.8 cross compiler binaries + when: cross_gcc48_download.changed ansible.builtin.command: argv: ['tar', '-C', '/', '-xzf', '/root/gcc-4.8-x86_64-linux-gnu-cross.tgz'] + + - name: Download gcc-5.5 cross compiler binaries + ansible.builtin.get_url: + url: https://obj.internal.efficios.com/jenkins/gcc-5.5-x86_64-linux-gnu-cross.tgz + dest: /root/gcc-5.5-x86_64-linux-gnu-cross.tgz + register: cross_gcc55_download + - name: Extract gcc-5.5 cross compiler binaries + when: cross_gcc55_download.changed + ansible.builtin.command: + argv: ['tar', '-xzf', '/root/gcc-5.5-x86_64-linux-gnu-cross.tgz', '-C', '/'] diff --git a/scripts/lttng-modules/param-build.sh b/scripts/lttng-modules/param-build.sh index 1535b3e..07324e7 100644 --- a/scripts/lttng-modules/param-build.sh +++ b/scripts/lttng-modules/param-build.sh @@ -151,9 +151,27 @@ select_compiler() { exit 0 fi + if [ "${cross_arch}" = "arm64" ] && verlt "${kversion}" "3.7"; then + echo "ARM64 support was added as of v3.7. Don't try to build it." + exit 0 + fi + + if [ "${cross_arch}" = "arm64" ] && verlt "${kversion}" "3.18"; then + echo "lttng-modules requires gcc >= 5.1 for ARM64 due to compiler bugs in gcc." + echo "gcc-5 support was added to the kernel as of v3.18. Don't this to build it." + exit 0 + fi + if { verlt "$kversion" "4.4"; }; then # Force gcc-4.8 for kernels before 4.4 selected_cc='gcc-4.8' + # Due to compiler bugs in gcc on arm64, lttng-modules disallows + # compilation with gcc < 5.1. + if [[ "${cross_arch}" == "arm64" ]] ; then + selected_cc='gcc-5.5' + export PATH="${PATH:-}:/usr/local/gcc5.5/bin" + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:-}:/usr/local/gcc5.5/lib" + fi selected_cc_version=$(echo "${selected_cc}" | cut -d'-' -f2) else for cc in $(list_gccs) ; do @@ -200,7 +218,7 @@ export_kbuild_flags() { _KCPPFLAGS=() _HOSTCFLAGS=() - if [ "$selected_cc" != "gcc-4.8" ]; then + if { vergte "$selected_cc_version" "6"; }; then # Older kernel Makefiles do not expect the compiler to default to PIE _KAFLAGS+=(-fno-pie) _KCFLAGS+=(