X-Git-Url: https://git.liburcu.org/?a=blobdiff_plain;f=scripts%2Flttng-modules%2Fparam-build.sh;h=81389bd3b278421c178a068b769f0d7220753315;hb=1490b049e5e289235463b2eb3968abf3fc88d651;hp=18c165f678b240fa2732e74a8d9148113adbb0e6;hpb=c45b04df52b71069d2a2820dfacf5ca32291250b;p=lttng-ci.git diff --git a/scripts/lttng-modules/param-build.sh b/scripts/lttng-modules/param-build.sh index 18c165f..81389bd 100644 --- a/scripts/lttng-modules/param-build.sh +++ b/scripts/lttng-modules/param-build.sh @@ -1,6 +1,6 @@ -#!/bin/bash -exu +#!/bin/bash # -# Copyright (C) 2016-2019 - Michael Jeanson +# Copyright (C) 2016-2019 Michael Jeanson # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,15 +15,26 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +set -exu + # Parameters -arch=${arch:-} +platforms=${platforms:-} +# Derive arch from label if it isn't set +if [ -z "${arch:-}" ] ; then + # Labels may be platform specific, eg. jammy-amd64, deb12-armhf + regex='[[:alnum:]]+-([[:alnum:]]+)' + if [[ "${platforms}" =~ ${regex} ]] ; then + arch="${BASH_REMATCH[1]}" + else + arch="${platforms:-}" + fi +fi cross_arch=${cross_arch:-} ktag=${ktag:-} kgitrepo=${kgitrepo:-} mversion=${mversion:-} mgitrepo=${mgitrepo:-} - ## FUNCTIONS ## # Kernel version compare functions @@ -32,6 +43,7 @@ verlte() { } verlt() { + # shellcheck disable=SC2015 [ "$1" = "$2" ] && return 1 || verlte "$1" "$2" } @@ -40,13 +52,22 @@ vergte() { } vergt() { + # shellcheck disable=SC2015 [ "$1" = "$2" ] && return 1 || vergte "$1" "$2" } git_clone_modules_sources() { mkdir -p "$MODULES_GIT_DIR" - git clone --depth=1 -b "${mversion}" "${mgitrepo}" "$MODULES_GIT_DIR" + + # If the version starts with "refs/", checkout the specific git ref, otherwise treat it + # as a branch name. + if [ "${mversion:0:5}" = "refs/" ]; then + git clone --no-tags --depth=1 "${mgitrepo}" "$MODULES_GIT_DIR" + (cd "$MODULES_GIT_DIR" && git fetch origin "${mversion}" && git checkout FETCH_HEAD) + else + git clone --no-tags --depth=1 -b "${mversion}" "${mgitrepo}" "$MODULES_GIT_DIR" + fi } # Checkout a shallow kernel tree of the specified tag @@ -82,6 +103,16 @@ tar_archive_obj() { cd - } +list_gccs() { + local gccs + gccs=() + IFS=: read -r -a path_array <<< "$PATH" + while read -r gcc ; do + gccs+=("$gcc") + done < <(find "${path_array[@]}" -maxdepth 1 -regex '.*/gcc-[0-9\.]+$' -printf '%f\n' | sort -t- -k2 -V -r) + echo "${gccs[@]}" +} + # Find the most recent GCC version supported by the kernel sources select_compiler() { local selected_cc @@ -92,7 +123,7 @@ select_compiler() { set +e - for cc in gcc-5 gcc-4.8; do + for cc in $(list_gccs) ; do if "${CROSS_COMPILE:-}${cc}" -I include/ -D__LINUX_COMPILER_H -D__LINUX_COMPILER_TYPES_H -E include/linux/compiler-gcc.h; then selected_cc="$cc" break @@ -140,8 +171,25 @@ build_linux_kernel() { exit 0 fi fi + + # Disable riscv64 config generation, we don't have a toolchain on bionic + sed -i 's/riscv64 //' debian.master/etc/kernelconfig + fakeroot debian/rules clean KW_DEFCONFIG_DIR=. - fakeroot debian/rules genconfigs KW_DEFCONFIG_DIR=. + + # Hack for kernel Ubuntu-hwe-5.8 + # The debian/control file is produced by the clean target above, so this + # check needs to happen afterwards. + if [ ! -f "debian/compat" ] && ! grep -q debhelper-compat debian/control; then + echo "10" > "debian/compat" + fi + + # genconfigs check can fail in certain cases, eg. when a more recent + # compiler exposes kernel configuration flags which aren't set in the + # Ubuntu annotations. + # In any case, the configuration will be updated with any missing values + # later in our build script. + fakeroot debian/rules genconfigs KW_DEFCONFIG_DIR=. do_skip_checks=true cp CONFIGS/"${ubuntu_config}" .config ;; @@ -151,20 +199,15 @@ build_linux_kernel() { export ARCH="i386" fi - # allyesconfig is mostly broken for kernels of the 2.6 series - if verlt "$kversion" "3.0"; then - vanilla_config="defconfig" - fi - make "${vanilla_config}" ;; esac - # silentoldconfig was renamed in 4.19 + # oldnoconfig was renamed in 4.19 if vergte "$kversion" "4.19"; then - update_conf_target="syncconfig" + update_conf_target="olddefconfig" else - update_conf_target="silentoldconfig" + update_conf_target="oldnoconfig" fi # Fix 'defined(@array)' was removed from recent perl @@ -173,7 +216,7 @@ build_linux_kernel() { fi # Fix syntax of inline assembly which is confused with C++11 raw strings on gcc >= 5 - if [ "$CC" != "gcc-4.8" ]; then + if [ "$HOSTCC" != "gcc-4.8" ]; then if [ -f "arch/x86/kvm/svm.c" ]; then sed -i 's/ R"/ R "/g; s/"R"/" R "/g' arch/x86/kvm/svm.c fi @@ -183,11 +226,25 @@ build_linux_kernel() { fi fi + # Newer binutils don't accept 3 operand 'cmp' instructions on ppc64 + # Convert them to 'cmpw' which was previously done silently + if verlt "$kversion" "4.9"; then + find arch/powerpc/ -name "*.S" -print0 | xargs -0 sed -i "s/\(cmp\)\(\s\+[a-zA-Z0-9]\+,\s*[a-zA-Z0-9]\+,\s*[a-zA-Z0-9]\+\)/cmpw\2/" + find arch/powerpc/ -name "*.S" -print0 | xargs -0 sed -i "s/\(cmpli\)\(\s\+[a-zA-Z0-9]\+,\s*[a-zA-Z0-9]\+,\s*[a-zA-Z0-9]\+\)/cmplwi\2/" + sed -i "s/\$pie \-o \"\$ofile\"/\$pie --no-dynamic-linker -o \"\$ofile\"/" arch/powerpc/boot/wrapper + fi + # Fix a typo in v2.6.36.x if [ -f "arch/x86/kernel/entry_64.S" ]; then sed -i 's/END(do_hypervisor_callback)/END(xen_do_hypervisor_callback)/' arch/x86/kernel/entry_64.S fi + # Fix compiler switch in vdso Makefile for 2.6.36 to 2.6.36.2 + if { vergte "$kversion" "2.6.36" && verlte "$kversion" "2.6.36.3"; }; then + sed -i 's/-m elf_x86_64/-m64/' arch/x86/vdso/Makefile + sed -i 's/-m elf_i386/-m32/' arch/x86/vdso/Makefile + fi + # Fix kernel < 3.0 with gcc >= 4.7 if verlt "$kversion" "3.0"; then sed -i '/linux\/compiler.h/a #include \/* For asmregparm *\/' arch/x86/include/asm/ptrace.h @@ -197,37 +254,56 @@ build_linux_kernel() { fi # GCC 4.8 - sed -i "s/CONFIG_CC_STACKPROTECTOR_STRONG=y/# CONFIG_CC_STACKPROTECTOR_STRONG is not set/g" .config + if [ "$HOSTCC" == "gcc-4.8" ]; then + scripts/config --disable CONFIG_CC_STACKPROTECTOR_STRONG + fi # Don't try to sign modules - sed -i "s/CONFIG_MODULE_SIG=y/# CONFIG_MODULE_SIG is not set/g" .config + scripts/config --disable CONFIG_MODULE_SIG # Disable kernel stack frame correctness validation, introduced in 4.6.0 and currently fails - sed -i "s/CONFIG_STACK_VALIDATION=y/# CONFIG_STACK_VALIDATION is not set/g" .config + scripts/config --disable CONFIG_STACK_VALIDATION # Cause problems with inline assembly on i386 - sed -i "s/CONFIG_DEBUG_SECTION_MISMATCH=y/# CONFIG_DEBUG_SECTION_MISMATCH is not set/g" .config + scripts/config --disable CONFIG_DEBUG_SECTION_MISMATCH # Don't build samples, they are broken on some kernel releases - sed -i "s/CONFIG_SAMPLES=y/# CONFIG_SAMPLES is not set/g" .config - sed -i "s/CONFIG_BUILD_DOCSRC=y/# CONFIG_BUILD_DOCSRC is not set/g" .config + scripts/config --disable CONFIG_SAMPLES + scripts/config --disable CONFIG_BUILD_DOCSRC # Disable kcov - sed -i "s/CONFIG_KCOV=y/# CONFIG_KCOV is not set/g" .config + scripts/config --disable CONFIG_KCOV # Broken on some RT kernels - sed -i "s/CONFIG_HYPERV=y/# CONFIG_HYPERV is not set/g" .config + scripts/config --disable CONFIG_HYPERV + + # Broken drivers + scripts/config --disable CONFIG_RAPIDIO_TSI721 + scripts/config --disable CONFIG_SGI_XP + scripts/config --disable CONFIG_MFD_WM8994 + scripts/config --disable CONFIG_DRM_RADEON + scripts/config --disable CONFIG_SND_SOC_WM5100 # IGBVF won't build with recent gcc on 2.6.38.x if { vergte "$kversion" "2.6.37" && verlt "$kversion" "2.6.38"; }; then - sed -i "s/CONFIG_IGBVF=y/# CONFIG_IGBVF is not set/g" .config + scripts/config --disable CONFIG_IGBVF fi + # Don't fail the build on warnings + scripts/config --disable CONFIG_WERROR + # Set required options - sed -i 's/# CONFIG_KPROBES is not set/CONFIG_KPROBES=y/g' .config - sed -i 's/# CONFIG_FTRACE is not set/CONFIG_FTRACE=y/g' .config - sed -i 's/# CONFIG_BLK_DEV_IO_TRACE is not set/CONFIG_BLK_DEV_IO_TRACE=y/g' .config - sed -i 's/# CONFIG_KALLSYMS_ALL is not set/CONFIG_KALLSYMS_ALL=y/g' .config + scripts/config --enable CONFIG_TRACEPOINTS + scripts/config --enable CONFIG_KALLSYMS + scripts/config --enable CONFIG_HIGH_RES_TIMERS + scripts/config --enable CONFIG_KPROBES + scripts/config --enable CONFIG_FTRACE + scripts/config --enable CONFIG_BLK_DEV_IO_TRACE + scripts/config --enable CONFIG_KALLSYMS_ALL + scripts/config --enable CONFIG_HAVE_SYSCALL_TRACEPOINTS + scripts/config --enable CONFIG_PERF_EVENTS + scripts/config --enable CONFIG_EVENT_TRACING + scripts/config --enable CONFIG_KRETPROBES # Debug #cat .config @@ -239,8 +315,8 @@ build_linux_kernel() { # Save the kernel and modules mkdir -p "$LINUX_INSTOBJ_DIR/boot" - make INSTALL_MOD_PATH="$LINUX_INSTOBJ_DIR" INSTALL_MOD_STRIP=1 modules_install - make INSTALL_PATH="$LINUX_INSTOBJ_DIR/boot" install + make INSTALL_MOD_PATH="$LINUX_INSTOBJ_DIR" INSTALL_MOD_STRIP=1 modules_install CC="$CC" + make INSTALL_MOD_PATH="$LINUX_INSTOBJ_DIR" INSTALL_PATH="$LINUX_INSTOBJ_DIR/boot" install CC="$CC" rm -f "$LINUX_INSTOBJ_DIR/lib/modules/${krelease}/source" "$LINUX_INSTOBJ_DIR/lib/modules/${krelease}/build" ln -s ../../../../sources "$LINUX_INSTOBJ_DIR/lib/modules/${krelease}/source" ln -s ../../../../sources "$LINUX_INSTOBJ_DIR/lib/modules/${krelease}/source" @@ -282,9 +358,13 @@ extract_distro_headers() { rm -f "${LINUX_HDROBJ_DIR}/scripts/*.o" rm -f "${LINUX_HDROBJ_DIR}/scripts/*/*.o" - # On powerpc this object is required to link modules + # On powerpc 32bits this object is required to link modules if [ "${karch}" = "powerpc" ]; then - cp -a --parents arch/powerpc/lib/crtsavres.[So] "${LINUX_HDROBJ_DIR}/" + if [ "x$(scripts/config -s CONFIG_PPC64)" = "xy" ] && vergte "${kversion}" "5.4"; then + : + else + cp -a --parents arch/powerpc/lib/crtsavres.[So] "${LINUX_HDROBJ_DIR}/" + fi fi # On arm64 between 4.13 and 1.15 this object is required to build with ftrace support @@ -332,7 +412,9 @@ extract_distro_headers() { touch -r "${LINUX_HDROBJ_DIR}/.config" "${LINUX_HDROBJ_DIR}/include/generated/autoconf.h" # Copy .config to include/config/auto.conf so "make prepare" is unnecessary. - cp "${LINUX_HDROBJ_DIR}/.config" "${LINUX_HDROBJ_DIR}/include/config/auto.conf" + if [ ! -f "${LINUX_HDROBJ_DIR}/include/config/auto.conf" ]; then + cp "${LINUX_HDROBJ_DIR}/.config" "${LINUX_HDROBJ_DIR}/include/config/auto.conf" + fi # Finally clean the object files from the full source tree make clean @@ -340,9 +422,13 @@ extract_distro_headers() { # And regen the modules support files make modules_prepare CC="$CC" - # On powerpc this object is required to link modules + # On powerpc 32bits this object is required to link modules if [ "${karch}" = "powerpc" ]; then - make arch/powerpc/lib/crtsavres.o CC="$CC" + if [ "x$(scripts/config -s CONFIG_PPC64)" = "xy" ] && vergte "${kversion}" "5.4"; then + : + else + make arch/powerpc/lib/crtsavres.o CC="$CC" + fi fi # On arm64 between 4.13 and 4.15 this object is required to build with ftrace support @@ -372,6 +458,10 @@ build_modules() { kversion=$(make -C "$LINUX_HDROBJ_DIR" -s kernelversion) + # Try to catch some compatibility problems by turning some + # warnings into errors. + #export KCFLAGS="$KCFLAGS -Wall -Werror" + # Enter lttng-modules source dir cd "${MODULES_GIT_DIR}" @@ -384,11 +474,12 @@ build_modules() { # Build modules KERNELDIR="${kdir}" make -j"${NPROC}" V=1 CC="$CC" + ret=$? set -e # We expect this build to fail, if it doesn't, fail the job. - if [ "$?" -eq 0 ]; then + if [ "$ret" -eq 0 ]; then echo "This build should have failed." exit 1 fi @@ -396,8 +487,6 @@ build_modules() { # We have to publish at least one file or the build will fail echo "This kernel is broken, there is a deadlock in the timekeeping subsystem." > "${outdir}/BROKEN.txt.ko" - set -e - KERNELDIR="${kdir}" make clean else # Regular build @@ -448,14 +537,14 @@ if [ "x${cross_arch}" != "x" ]; then "armhf") karch="arm" cross_compile="arm-linux-gnueabihf-" - vanilla_config="allyesconfig" + vanilla_config="imx_v6_v7_defconfig" ubuntu_config="armhf-config.flavour.generic" ;; "arm64") karch="arm64" cross_compile="aarch64-linux-gnu-" - vanilla_config="allyesconfig" + vanilla_config="defconfig" ubuntu_config="arm64-config.flavour.generic" ;; @@ -489,37 +578,37 @@ elif [ "x${arch}" != "x" ]; then case "$arch" in "i386") karch="x86" - vanilla_config="allyesconfig" + vanilla_config="allmodconfig" ubuntu_config="i386-config.flavour.generic" ;; "amd64") karch="x86" - vanilla_config="allyesconfig" + vanilla_config="allmodconfig" ubuntu_config="amd64-config.flavour.generic" ;; "armhf") karch="arm" - vanilla_config="allyesconfig" + vanilla_config="allmodconfig" ubuntu_config="armhf-config.flavour.generic" ;; "arm64") karch="arm64" - vanilla_config="allyesconfig" + vanilla_config="allmodconfig" ubuntu_config="arm64-config.flavour.generic" ;; "powerpc") karch="powerpc" - vanilla_config="allyesconfig" + vanilla_config="allmodconfig" ubuntu_config="powerpc-config.flavour.powerpc-smp" ;; "ppc64el") karch="powerpc" - vanilla_config="allyesconfig" + vanilla_config="allmodconfig" ubuntu_config="ppc64el-config.flavour.generic" ;; @@ -529,7 +618,7 @@ elif [ "x${arch}" != "x" ]; then ;; esac else - echo "Not arch or cross_arch specified" + echo "No arch or cross_arch specified" exit 1 fi @@ -553,16 +642,26 @@ signature_v2 = False" > "$WORKSPACE/.s3cfg" url_hash="$(echo -n "$kgitrepo" | md5sum | awk '{ print $1 }')" obj_name="linux.tar.bz2" -obj_url_prefix="$OBJ_STORE_URL/linux-build/$url_hash/$ktag/$arch/${cross_arch:-native}" + +if [ "x${cross_arch}" = "x" ]; then + obj_url_prefix="$OBJ_STORE_URL/linux-build/$url_hash/$ktag/platform-${platforms}/$arch/native" +else + obj_url_prefix="$OBJ_STORE_URL/linux-build/$url_hash/$ktag/platform-${platforms}/${cross_arch}" +fi + obj_url="$obj_url_prefix/$obj_name" set +e -s3cmd -c "$WORKSPACE/.s3cfg" get "$obj_url" +# In s3cmd 2.3, the return code of get when an object does not exist (64) +# is different than in 2.2 (12). The return codes of 's3cmd info' are +# consistent between 2.2 and 2.3. +s3cmd -c "$WORKSPACE/.s3cfg" info "$obj_url" ret=$? set -e case "$ret" in "0") + s3cmd -c "$WORKSPACE/.s3cfg" get "$obj_url" extract_archive_obj ;;