scripts/lttng-modules/param-build.sh

   1 #!/bin/bash
   2 #
   3 # Copyright (C) 2016-2019 Michael Jeanson <mjeanson@efficios.com>
   4 #
   5 # This program is free software: you can redistribute it and/or modify
   6 # it under the terms of the GNU General Public License as published by
   7 # the Free Software Foundation, either version 3 of the License, or
   8 # (at your option) any later version.
   9 #
  10 # This program is distributed in the hope that it will be useful,
  11 # but WITHOUT ANY WARRANTY; without even the implied warranty of
  12 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  13 # GNU General Public License for more details.
  14 #
  15 # You should have received a copy of the GNU General Public License
  16 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18 set -exu
  19
  20 # Parameters
  21 platforms=${platforms:-}
  22 # Derive arch from label if it isn't set
  23 if [ -z "${arch:-}" ] ; then
  24     # Labels may be platform specific, eg. jammy-amd64, deb12-armhf
  25     regex='[[:alnum:]]+-([[:alnum:]]+)'
  26     if [[ "${platforms}" =~ ${regex} ]] ; then
  27         arch="${BASH_REMATCH[1]}"
  28     else
  29         arch="${platforms:-}"
  30     fi
  31 fi
  32
  33 cross_arch=${cross_arch:-}
  34 ktag=${ktag:-}
  35 kgitrepo=${kgitrepo:-}
  36 mversion=${mversion:-}
  37 mgitrepo=${mgitrepo:-}
  38 make_args=()
  39
  40 DEBUG=${DEBUG:-}
  41
  42 ## FUNCTIONS ##
  43
  44 # Kernel version compare functions
  45 verlte() {
  46     [  "$1" = "$(printf '%s\n%s' "$1" "$2" | sort -V | head -n1)" ]
  47 }
  48
  49 verlt() {
  50     # shellcheck disable=SC2015
  51     [ "$1" = "$2" ] && return 1 || verlte "$1" "$2"
  52 }
  53
  54 vergte() {
  55     [  "$1" = "$(printf '%s\n%s' "$1" "$2" | sort -V | tail -n1)" ]
  56 }
  57
  58 vergt() {
  59     # shellcheck disable=SC2015
  60     [ "$1" = "$2" ] && return 1 || vergte "$1" "$2"
  61 }
  62
  63 git_clone_modules_sources() {
  64     mkdir -p "$MODULES_GIT_DIR"
  65
  66     # If the version starts with "refs/", checkout the specific git ref, otherwise treat it
  67     # as a branch name.
  68     if [ "${mversion:0:5}" = "refs/" ]; then
  69         git clone --no-tags --depth=1 "${mgitrepo}" "$MODULES_GIT_DIR"
  70         (cd "$MODULES_GIT_DIR" && git fetch origin "${mversion}" && git checkout FETCH_HEAD)
  71     else
  72         git clone --no-tags --depth=1 -b "${mversion}" "${mgitrepo}" "$MODULES_GIT_DIR"
  73     fi
  74 }
  75
  76 # Checkout a shallow kernel tree of the specified tag
  77 git_clone_linux_sources() {
  78     mkdir -p "$LINUX_GIT_DIR"
  79     git clone --depth=1 -b "${ktag}" --reference "$LINUX_GIT_REF_REPO_DIR" "${kgitrepo}" "$LINUX_GIT_DIR"
  80 }
  81
  82
  83 # Export the kernel sources from the git repo
  84 git_export_linux_sources() {
  85     cd "$LINUX_GIT_DIR"
  86     git archive "${ktag}" | tar -x -C "$LINUX_SRCOBJ_DIR"
  87 }
  88
  89
  90 # Upload the tar archive to the object store
  91 upload_archive_obj() {
  92     s3cmd -c "$WORKSPACE/.s3cfg" put "$WORKSPACE/$obj_name" "$obj_url_prefix/"
  93     rm -f "$WORKSPACE/$obj_name"
  94 }
  95
  96
  97 extract_archive_obj() {
  98     tar -xf "$WORKSPACE/$obj_name" -C "$LINUX_OBJ_DIR" -I pbzip2
  99     rm -f "$WORKSPACE/$obj_name"
 100 }
 101
 102
 103 tar_archive_obj() {
 104     cd "$LINUX_OBJ_DIR"
 105     tar -cf "$WORKSPACE/$obj_name" -I pbzip2 .
 106     cd -
 107 }
 108
 109 list_gccs() {
 110     local gccs
 111     gccs=()
 112     IFS=: read -r -a path_array <<< "$PATH"
 113     while read -r gcc ; do
 114         gccs+=("$gcc")
 115     done < <(find "${path_array[@]}" -maxdepth 1 -regex '.*/gcc-[0-9\.]+$' -printf '%f\n' | sort -t- -k2 -V -r)
 116     echo "${gccs[@]}"
 117 }
 118
 119 # Find the most recent GCC version supported by the kernel sources
 120 select_compiler() {
 121     local selected_cc
 122
 123     cd "$LINUX_SRCOBJ_DIR"
 124
 125     kversion=$(make -s kernelversion)
 126
 127     set +e
 128
 129     for cc in $(list_gccs) ; do
 130         if "${CROSS_COMPILE:-}${cc}" -I include/ -D__LINUX_COMPILER_H -D__LINUX_COMPILER_TYPES_H -E include/linux/compiler-gcc.h; then
 131             cc_version=$(echo "${cc}" | cut -d'-' -f2)
 132             if { verlt "${kversion}" "5.17"; } && { vergt "${cc_version}" "11"; } ; then
 133                 # Using gcc-12+ with '-Wuse-after-free' breaks the build of older
 134                 # kernels (in particular, objtool). Some releases on LTS
 135                 # branches between 4.x and 5.15 can be built with gcc-12.
 136                 # @see https://lore.kernel.org/lkml/20494.1643237814@turing-police/
 137                 # @see https://gitlab.com/linux-kernel/stable/-/commit/52a9dab6d892763b2a8334a568bd4e2c1a6fde66
 138                 continue
 139             fi
 140         selected_cc="$cc"
 141         break
 142       fi
 143     done
 144
 145     set -e
 146
 147     # Force gcc-4.8 for kernels before 4.4
 148     if { verlt "$kversion" "4.4"; }; then
 149         selected_cc='gcc-4.8'
 150     fi
 151
 152     if [ "x$selected_cc" = "x" ]; then
 153       echo "Found no suitable compiler."
 154       exit 1
 155     fi
 156
 157     _KAFLAGS=()
 158     _KCFLAGS=()
 159     _KCPPFLAGS=()
 160     _HOSTCFLAGS=()
 161     if [ "$selected_cc" != "gcc-4.8" ]; then
 162         # Older kernel Makefiles do not expect the compiler to default to PIE
 163         _KAFLAGS+=(-fno-pie)
 164         _KCFLAGS+=(
 165             -fno-pie
 166             -no-pie
 167             -fno-stack-protector
 168         )
 169         _KCPPFLAGS+=(-fno-pie)
 170     fi
 171
 172     selected_cc_version="$(echo "${selected_cc}" | cut -d'-' -f2)"
 173     if { vergte "${selected_cc_version}" "10"; } && { verlt "${kversion}" "5.10"; } ; then
 174         # gcc-10 changed the default from '-fcommon' to '-fno-common', which
 175         # causes a linker failure. '-fcommon' can be set on the HOSTCFLAGS
 176         # to avoid the issue.
 177         # @see https://gitlab.com/linux-kernel/stable/-/commit/e33a814e772cdc36436c8c188d8c42d019fda639
 178         _HOSTCFLAGS+=(-fcommon)
 179     fi
 180
 181     if [ "${cross_arch:-}" == "armhf" ] ; then
 182         if { verlt "${kversion}" "5.14"; } ; then
 183             # Work-around for producing instructions that aren't valid for the
 184             # default architectures.
 185             # Eg. Error: selected processor does not support `cpsid i' in ARM mode
 186             _KCFLAGS+=(-march=armv7-a -mfpu=vfpv3-d16)
 187             _KCPPFLAGS+=(-march=armv7-a -mfpu=vfpv3-d16)
 188         fi
 189     fi
 190
 191     if { vergt "${selected_cc_version}" "8"; } && { vergte "${kversion}" "4.15"; } && { verlt "${kversion}" "4.17"; } ; then
 192         # This was added to -Wall in gcc 9 but some kernels do not include the fixes
 193         # @see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88583
 194         _KCFLAGS+=(-Wno-error=packed-not-aligned)
 195     fi
 196
 197     export KAFLAGS="${_KAFLAGS[*]}"
 198     export KCFLAGS="${_KCFLAGS[*]}"
 199     export KCPPFLAGS="${_KCPPFLAGS[*]}"
 200     export HOSTCFLAGS="${_HOSTCFLAGS[*]}"
 201     export CC="${CROSS_COMPILE:-}${selected_cc}"
 202     export HOSTCC="${selected_cc}"
 203
 204     make_args=(
 205         CC="${CC}"
 206         HOSTCC="${HOSTCC}"
 207         HOSTCFLAGS="${HOSTCFLAGS:-}"
 208     )
 209     if [ -n "${DEBUG}" ] ; then
 210         make_args+=(
 211             V=1
 212         )
 213     fi
 214     cd -
 215 }
 216
 217 patch_linux_kernel() {
 218     local commit_hash
 219     commit_hash="$1"
 220     set +e
 221     git -C "${LINUX_GIT_REF_REPO_DIR}" format-patch -n1 --stdout "${commit_hash}" | patch -p1
 222     set -e
 223     if [ "$?" -gt 1 ] ; then
 224         echo "Serious issue patching"
 225         exit 1
 226     fi
 227 }
 228
 229 build_linux_kernel() {
 230     cd "$LINUX_SRCOBJ_DIR"
 231
 232     kversion=$(make -s kernelversion "${make_args[@]}")
 233
 234     if { verlt "${kversion}" "3.3"; } && [ "${vanilla_config}" = "imx_v6_v7_defconfig" ] ; then
 235         # imx_v6_v7 didn't exist before 06965c39b4c63933fa0a1cde2237ef85477c5655
 236         if { verlt "${kversion}" "3.2"; } ; then
 237             vanilla_config='mx5_defconfig'
 238         else
 239             vanilla_config='mx51_defconfig'
 240         fi
 241     fi
 242
 243     if { verlt "${kversion}" "3.13"; } && [ "${vanilla_config}" = "pseries_le_defconfig" ] ; then
 244         # pseries_le_deconfig was introduced in f53e462e907cbaed29c49c0f10f5b8f614e1bf1d
 245         vanilla_config='pseries_defconfig'
 246     fi
 247
 248     # Generate kernel configuration
 249     case "$ktag" in
 250       Ubuntu*)
 251         if [ "${cross_arch}" = "powerpc" ]; then
 252           if vergte "${kversion}" "4.10"; then
 253             echo "Ubuntu removed big endian powerpc configuration from kernel >= 4.10. Don't try to build it."
 254             exit 0
 255           fi
 256         fi
 257
 258         # Disable riscv64 config generation, we don't have a toolchain on bionic
 259         sed -i 's/riscv64 //' debian.master/etc/kernelconfig
 260
 261         fakeroot debian/rules clean KW_DEFCONFIG_DIR=.
 262
 263         # Hack for kernel Ubuntu-hwe-5.8
 264         # The debian/control file is produced by the clean target above, so this
 265         # check needs to happen afterwards.
 266         if [ ! -f "debian/compat" ] && ! grep -q debhelper-compat debian/control; then
 267             echo "10" > "debian/compat"
 268         fi
 269
 270         # genconfigs check can fail in certain cases, eg. when a more recent
 271         # compiler exposes kernel configuration flags which aren't set in the
 272         # Ubuntu annotations.
 273         # In any case, the configuration will be updated with any missing values
 274         # later in our build script.
 275         fakeroot debian/rules genconfigs KW_DEFCONFIG_DIR=. do_skip_checks=true
 276         cp CONFIGS/"${ubuntu_config}" .config
 277         ;;
 278
 279       *)
 280         # Force 32bit build on i386, default is 64bit
 281         if [ "$arch" = "i386" ]; then
 282             export ARCH="i386"
 283         fi
 284
 285         make "${vanilla_config}" "${make_args[@]}"
 286         ;;
 287     esac
 288
 289     if [ "${vanilla_config}" = "pseries_defconfig" ] && [ "${cross_arch}" = "ppc64el" ] ; then
 290         # @see diff <(git show v3.13:arch/powerpc/configs/pseries_defconfig) <(git show v3.13:arch/powerpc/configs/pseries_le_defconfig)
 291         scripts/config --enable CONFIG_CPU_LITTLE_ENDIAN
 292         scripts/config --enable CONFIG_CMA
 293         scripts/config --disable CONFIG_XMON_DEFAULT
 294         # scripts/config --disable CONFIG_VIRTUALIZATION
 295         # scripts/config --disable CONFIG_KVM_BOOK3S_64
 296         scripts/config --disable CONFIG_KVM_BOOK3S_64_HV
 297     fi
 298
 299     # oldnoconfig was renamed in 4.19
 300     if vergte "$kversion" "4.19"; then
 301         update_conf_target="olddefconfig"
 302     else
 303         update_conf_target="oldnoconfig"
 304     fi
 305
 306     # Fix 'defined(@array)' was removed from recent perl
 307     if [ -f "kernel/timeconst.pl" ]; then
 308       sed -i 's/defined(\@\(.*\))/@\1/' kernel/timeconst.pl
 309     fi
 310
 311     # Fix syntax of inline assembly which is confused with C++11 raw strings on gcc >= 5
 312     if [ "$HOSTCC" != "gcc-4.8" ]; then
 313       if [ -f "arch/x86/kvm/svm.c" ]; then
 314         sed -i 's/ R"/ R "/g; s/"R"/" R "/g' arch/x86/kvm/svm.c
 315       fi
 316
 317       if [ -f "arch/x86/kvm/vmx.c" ]; then
 318         sed -i 's/ R"/ R "/g; s/"R"/" R "/g' arch/x86/kvm/vmx.c
 319       fi
 320     fi
 321
 322     if { verlt "${kversion}" "5.11"; } && { vergte "${kversion}" "4.10"; } ; then
 323         # Binutils > 2.35 strips empty symbol tables, causing obltool to fail
 324         # in certain cases when files are empty.
 325         # @see https://gitlab.com/linux-kernel/stable/-/commit/1d489151e9f9d1647110277ff77282fe4d96d09b
 326         #
 327         # There doesn't seem to be any LD/AS/AR flags to control this behaviour,
 328         # therefore patching tools/objtool/elf.c is attempted.
 329         patch_linux_kernel 1d489151e9f9d1647110277ff77282fe4d96d09b
 330         if { verlt "${kversion}" "4.18"; } ; then
 331             patch_linux_kernel e81e0724432542af8d8c702c31e9d82f57b1ff31
 332         fi
 333     fi
 334
 335     if { vergt "${selected_cc_version}" "7"; } && { vergte "${kversion}" "4.15"; } && { verlt "${kversion}" "4.17"; } ; then
 336         # Builds fail due to -Werror=restrict in pager.o and str_error_r.o
 337         if { verlt "${kversion}" "4.16"; } ; then
 338             # This is patched since objtool's Makefile doesn't respect HOSTCFLAGS
 339             # @see https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=ad343a98e74e85aa91d844310e797f96fee6983b
 340             patch_linux_kernel ad343a98e74e85aa91d844310e797f96fee6983b
 341         fi
 342         if { verlt "${kversion}" "4.17"; } ; then
 343             # This is patched since objtool's Makefile doesn't respect HOSTCFLAGS
 344             # @see https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=854e55ad289ef8888e7991f0ada85d5846f5afb9
 345             patch_linux_kernel 854e55ad289ef8888e7991f0ada85d5846f5afb9
 346         fi
 347
 348         # More recent compiler optimizations expose build errors with netronome.
 349         # It seems easier to disable the driver than to attempt patching.
 350         scripts/config --disable CONFIG_NET_VENDOR_NETRONOME
 351         scripts/config --disable CONFIG_MICREL_PHY
 352
 353         # Duplicate decalarations of __force_order
 354         # @see https://gitlab.com/linux-kernel/stable/-/commit/df6d4f9db79c1a5d6f48b59db35ccd1e9ff9adfc
 355         # However, kaslr_64.c doesn't exit in 4.15, 4.16, it's named pagetable.c
 356         if [ -f arch/x86/boot/compressed/pagetable.c ] ; then
 357             sed -i '/^unsigned long __force_order;$/d' arch/x86/boot/compressed/pagetable.c
 358         fi
 359     fi
 360
 361     if { vergte "${kversion}" "4.15"; } && { verlt "${kversion}" "4.18"; } ; then
 362         # Some old kernels fail to build when make is too new
 363         # @see https://gitlab.com/linux-kernel/stable/-/commit/9feeb638cde083c737e295c0547f1b4f28e99583
 364         patch_linux_kernel 9564a8cf422d7b58f6e857e3546d346fa970191e
 365         # @see https://gitlab.com/linux-kernel/stable/-/commit/9feeb638cde083c737e295c0547f1b4f28e99583
 366         patch_linux_kernel 9feeb638cde083c737e295c0547f1b4f28e99583
 367     fi
 368
 369     if { vergte "${kversion}" "4.12"; } && { verlt "${kversion}" "4.18"; } ; then
 370         # Old kernels can fail to build while on newer host kernels
 371         # @see https://gitlab.com/linux-kernel/stable/-/commit/dfbd199a7cfe3e3cd8531e1353cdbd7175bfbc5e
 372         #
 373         patch_linux_kernel dfbd199a7cfe3e3cd8531e1353cdbd7175bfbc5e
 374     fi
 375
 376     if { vergte "${kversion}" "3.18"; } && { verlt "${kversion}" "4.4"; } ; then
 377         # Compatibility with binutils >= ~ 2.31
 378         patch_linux_kernel b21ebf2fb4cde1618915a97cc773e287ff49173e
 379     fi
 380
 381     # The above patch only partially applies linux 3.17, and has been, so a
 382     # rebased version is used instead.
 383     if { vergte "${kversion}" "3.17"; } && { verlt "${kversion}" "3.18"; } ; then
 384         cat <<'EOF' | patch -p1
 385 diff --git a/arch/x86/kernel/machine_kexec_64.c b/arch/x86/kernel/machine_kexec_64.c
 386 index 48598105..0652c5b6 100644
 387 --- a/arch/x86/kernel/machine_kexec_64.c
 388 +++ b/arch/x86/kernel/machine_kexec_64.c
 389 @@ -516,6 +516,7 @@ int arch_kexec_apply_relocations_add(const Elf64_Ehdr *ehdr,
 390                                 goto overflow;
 391                         break;
 392                 case R_X86_64_PC32:
 393 +               case R_X86_64_PLT32:
 394                         value -= (u64)address;
 395                         *(u32 *)location = value;
 396                         break;
 397 diff --git a/arch/x86/kernel/module.c b/arch/x86/kernel/module.c
 398 index e69f9882..7c6bc9fe 100644
 399 --- a/arch/x86/kernel/module.c
 400 +++ b/arch/x86/kernel/module.c
 401 @@ -180,6 +180,7 @@ int apply_relocate_add(Elf64_Shdr *sechdrs,
 402                                 goto overflow;
 403                         break;
 404                 case R_X86_64_PC32:
 405 +               case R_X86_64_PLT32:
 406                         val -= (u64)loc;
 407                         *(u32 *)loc = val;
 408  #if 0
 409 diff --git a/arch/x86/tools/relocs.c b/arch/x86/tools/relocs.c
 410 index bbb1d225..8deeacbc 100644
 411 --- a/arch/x86/tools/relocs.c
 412 +++ b/arch/x86/tools/relocs.c
 413 @@ -763,6 +763,7 @@ static int do_reloc64(struct section *sec, Elf_Rel *rel, ElfW(Sym) *sym,
 414         switch (r_type) {
 415         case R_X86_64_NONE:
 416         case R_X86_64_PC32:
 417 +       case R_X86_64_PLT32:
 418                 /*
 419                  * NONE can be ignored and PC relative relocations don't
 420                  * need to be adjusted.
 421 EOF
 422     fi
 423
 424     if ( { vergte "${kversion}" "3.15"; } && { verlt "${kversion}" "4.4"; } ) ||
 425        ( { vergte "${kversion}" "4.15"; } && { verlt "${kversion}" "4.17"; } ); then
 426         # While the original motivation of this patch is for fixing builds using
 427         # clang, the same error occurs between linux >= 3.15 and < 4.4, and in
 428         # 4.15, 4.16.
 429         #
 430         # This patch only partially applies due to changes in kernel/Makefile,
 431         # so a supplementary patch is needed
 432         #
 433         # Without this patch, builds fail with
 434         #   Cannot find symbol for section 2: .text.
 435         #   kernel/elfcore.o: failed
 436         #
 437         # @see https://github.com/linuxppc/issues/issues/388
 438         # @see https://lore.kernel.org/lkml/20201204165742.3815221-2-arnd@kernel.org/
 439         #
 440         patch_linux_kernel 6e7b64b9dd6d96537d816ea07ec26b7dedd397b9
 441         if grep -q elfcore.o kernel/Makefile ; then
 442             sed -i '/^.* += elfcore.o$/d' kernel/Makefile
 443         fi
 444     fi
 445
 446     # Newer binutils don't accept 3 operand 'cmp' instructions on ppc64
 447     # Convert them to 'cmpw' which was previously done silently
 448     if verlt "$kversion" "4.9"; then
 449             find arch/powerpc/ -name "*.S" -print0 | xargs -0 sed -i "s/\(cmp\)\(\s\+[a-zA-Z0-9]\+,\s*[a-zA-Z0-9]\+,\s*[a-zA-Z0-9]\+\)/cmpw\2/"
 450             find arch/powerpc/ -name "*.S" -print0 | xargs -0 sed -i "s/\(cmpli\)\(\s\+[a-zA-Z0-9]\+,\s*[a-zA-Z0-9]\+,\s*[a-zA-Z0-9]\+\)/cmplwi\2/"
 451             sed -i "s/\$pie \-o \"\$ofile\"/\$pie --no-dynamic-linker -o \"\$ofile\"/" arch/powerpc/boot/wrapper
 452     fi
 453
 454     # Fix a typo in v2.6.36.x
 455     if [ -f "arch/x86/kernel/entry_64.S" ]; then
 456       sed -i 's/END(do_hypervisor_callback)/END(xen_do_hypervisor_callback)/' arch/x86/kernel/entry_64.S
 457     fi
 458
 459     # Fix compiler switch in vdso Makefile for 2.6.36 to 2.6.36.2
 460     if { vergte "$kversion" "2.6.36" && verlte "$kversion" "2.6.36.3"; }; then
 461       sed -i 's/-m elf_x86_64/-m64/' arch/x86/vdso/Makefile
 462       sed -i 's/-m elf_i386/-m32/' arch/x86/vdso/Makefile
 463     fi
 464
 465     # Fix kernel < 3.0 with gcc >= 4.7
 466     if verlt "$kversion" "3.0"; then
 467       sed -i '/linux\/compiler.h/a #include <linux\/linkage.h> \/* For asmregparm *\/' arch/x86/include/asm/ptrace.h
 468       sed -i 's/extern long syscall_trace_enter/extern asmregparm long syscall_trace_enter/' arch/x86/include/asm/ptrace.h
 469       sed -i 's/extern void syscall_trace_leave/extern asmregparm void syscall_trace_leave/' arch/x86/include/asm/ptrace.h
 470       echo "header-y += linkage.h" >> include/linux/Kbuild
 471     fi
 472
 473     if [ "${cross_arch}" = "powerpc" ] ; then
 474         if { vergte "${kversion}" "4.15"; } && { verlt "${kversion}" "4.16"; } ; then
 475             # Avoid register errors such as
 476             # aes_generic.c:(.text+0x4e0): undefined reference to `_restgpr_31_x'
 477             # @see https://gitlab.com/linux-kernel/stable/-/commit/148b974deea927f5dbb6c468af2707b488bfa2de
 478             make_args+=(
 479                 CFLAGS_aes_generic.o=''
 480             )
 481         fi
 482     fi
 483
 484     # GCC 4.8
 485     if [ "$HOSTCC" == "gcc-4.8" ]; then
 486         scripts/config --disable CONFIG_CC_STACKPROTECTOR_STRONG
 487         scripts/config --disable CONFIG_PPC_OF_BOOT_TRAMPOLINE
 488     fi
 489
 490     # Don't try to sign modules
 491     scripts/config --disable CONFIG_MODULE_SIG
 492
 493     # Disable kernel stack frame correctness validation, introduced in 4.6.0 and currently fails
 494     scripts/config --disable CONFIG_STACK_VALIDATION
 495
 496     # Cause problems with inline assembly on i386
 497     scripts/config --disable CONFIG_DEBUG_SECTION_MISMATCH
 498
 499     # Don't build samples, they are broken on some kernel releases
 500     scripts/config --disable CONFIG_SAMPLES
 501     scripts/config --disable CONFIG_BUILD_DOCSRC
 502
 503     # Disable kcov
 504     scripts/config --disable CONFIG_KCOV
 505
 506     # Broken on some RT kernels
 507     scripts/config --disable CONFIG_HYPERV
 508
 509     # Broken drivers
 510     scripts/config --disable CONFIG_RAPIDIO_TSI721
 511     scripts/config --disable CONFIG_SGI_XP
 512     scripts/config --disable CONFIG_MFD_WM8994
 513     scripts/config --disable CONFIG_DRM_RADEON
 514     scripts/config --disable CONFIG_SND_SOC_WM5100
 515
 516     # IGBVF won't build with recent gcc on 2.6.38.x
 517     if { vergte "$kversion" "2.6.37" && verlt "$kversion" "2.6.38"; }; then
 518       scripts/config --disable CONFIG_IGBVF
 519     fi
 520
 521     # Don't fail the build on warnings
 522     scripts/config --disable CONFIG_WERROR
 523     scripts/config --enable CONFIG_PPC_DISABLE_WERROR
 524
 525     # Set required options
 526     scripts/config --enable CONFIG_TRACEPOINTS
 527     scripts/config --enable CONFIG_KALLSYMS
 528     scripts/config --enable CONFIG_HIGH_RES_TIMERS
 529     scripts/config --enable CONFIG_KPROBES
 530     scripts/config --enable CONFIG_FTRACE
 531     scripts/config --enable CONFIG_BLK_DEV_IO_TRACE
 532     scripts/config --enable CONFIG_KALLSYMS_ALL
 533     scripts/config --enable CONFIG_HAVE_SYSCALL_TRACEPOINTS
 534     scripts/config --enable CONFIG_PERF_EVENTS
 535     scripts/config --enable CONFIG_EVENT_TRACING
 536     scripts/config --enable CONFIG_KRETPROBES
 537
 538     if [ -n "${DEBUG}" ] ; then
 539         cat .config
 540     fi
 541
 542     make "$update_conf_target" "${make_args[@]}"
 543     make -j"$NPROC" "${make_args[@]}"
 544
 545     krelease=$(make -s kernelrelease "${make_args[@]}")
 546
 547     # Save the kernel and modules
 548     mkdir -p "$LINUX_INSTOBJ_DIR/boot"
 549     make INSTALL_MOD_PATH="$LINUX_INSTOBJ_DIR" INSTALL_MOD_STRIP=1 modules_install "${make_args[@]}"
 550     make INSTALL_MOD_PATH="$LINUX_INSTOBJ_DIR" INSTALL_PATH="$LINUX_INSTOBJ_DIR/boot" install "${make_args[@]}"
 551     rm -f "$LINUX_INSTOBJ_DIR/lib/modules/${krelease}/source" "$LINUX_INSTOBJ_DIR/lib/modules/${krelease}/build"
 552     ln -s ../../../../sources "$LINUX_INSTOBJ_DIR/lib/modules/${krelease}/source"
 553     ln -s ../../../../sources "$LINUX_INSTOBJ_DIR/lib/modules/${krelease}/source"
 554 }
 555
 556
 557 extract_distro_headers() {
 558
 559     # Enter linux source dir
 560     cd "${LINUX_SRCOBJ_DIR}"
 561
 562
 563     # For RT kernels, copy version file
 564     if [ -s localversion-rt ]; then
 565         cp -a localversion-rt "${LINUX_HDROBJ_DIR}"
 566     fi
 567
 568     # Copy all Makefile related stuff
 569     find . -path './include/*' -prune \
 570         -o -path './scripts/*' -prune -o -type f \
 571         \( -name 'Makefile*' -o -name 'Kconfig*' -o -name 'Kbuild*' -o \
 572             -name '*.sh' -o -name '*.pl' -o -name '*.lds' \) \
 573         -print | cpio -pd --preserve-modification-time "${LINUX_HDROBJ_DIR}"
 574
 575     # Copy base scripts and include dirs
 576     cp -a scripts include "${LINUX_HDROBJ_DIR}"
 577
 578     # Copy arch includes
 579     (find arch -name include -type d -print0 | \
 580         xargs -0 -n1 -i: find : -type f) | \
 581         cpio -pd --preserve-modification-time "${LINUX_HDROBJ_DIR}"
 582
 583     # Copy arch scripts
 584     (find arch -name scripts -type d -print0 | \
 585         xargs -0 -n1 -i: find : -type f) | \
 586         cpio -pd --preserve-modification-time "${LINUX_HDROBJ_DIR}"
 587
 588     # Cleanup scripts
 589     rm -f "${LINUX_HDROBJ_DIR}/scripts/*.o"
 590     rm -f "${LINUX_HDROBJ_DIR}/scripts/*/*.o"
 591
 592     # On powerpc 32bits this object is required to link modules
 593     if [ "${karch}" = "powerpc" ]; then
 594         if [ "x$(scripts/config -s CONFIG_PPC64)" = "xy" ] && vergte "${kversion}" "5.4"; then
 595             :
 596         else
 597             cp -a --parents arch/powerpc/lib/crtsavres.[So] "${LINUX_HDROBJ_DIR}/"
 598         fi
 599     fi
 600
 601     # On arm64 between 4.13 and 1.15 this object is required to build with ftrace support
 602     if [ "${karch}" = "arm64" ]; then
 603         if [ -f "arch/arm64/kernel/ftrace-mod.S" ]; then
 604             cp -a --parents arch/arm64/kernel/ftrace-mod.[So] "${LINUX_HDROBJ_DIR}/"
 605         fi
 606     fi
 607
 608     # Newer kernels need objtool to build modules when CONFIG_STACK_VALIDATION=y
 609     if [ -f tools/objtool/objtool ]; then
 610       cp -a --parents tools/objtool/objtool "${LINUX_HDROBJ_DIR}/"
 611     fi
 612
 613     if [ -f "arch/x86/kernel/macros.s" ]; then
 614       cp -a --parents arch/x86/kernel/macros.s "${LINUX_HDROBJ_DIR}/"
 615     fi
 616
 617     # Copy modules related stuff, if available
 618     if [ -s Module.symvers ]; then
 619         cp Module.symvers "${LINUX_HDROBJ_DIR}"
 620     fi
 621
 622     if [ -s System.map ]; then
 623         cp System.map "${LINUX_HDROBJ_DIR}"
 624     fi
 625
 626     if [ -s Module.markers ]; then
 627         cp Module.markers "${LINUX_HDROBJ_DIR}"
 628     fi
 629
 630     # Copy config file
 631     cp .config "${LINUX_HDROBJ_DIR}"
 632
 633     # Make sure the Makefile and version.h have a matching timestamp so that
 634     # external modules can be built
 635     if [ -s "${LINUX_HDROBJ_DIR}/include/generated/uapi/linux/version.h" ]; then
 636         touch -r "${LINUX_HDROBJ_DIR}/Makefile" "${LINUX_HDROBJ_DIR}/include/generated/uapi/linux/version.h"
 637     elif [ -s "${LINUX_HDROBJ_DIR}/include/linux/version.h" ]; then
 638         touch -r "${LINUX_HDROBJ_DIR}/Makefile" "${LINUX_HDROBJ_DIR}/include/linux/version.h"
 639     else
 640         echo "Missing version.h"
 641         exit 1
 642     fi
 643     touch -r "${LINUX_HDROBJ_DIR}/.config" "${LINUX_HDROBJ_DIR}/include/generated/autoconf.h"
 644
 645     # Copy .config to include/config/auto.conf so "make prepare" is unnecessary.
 646     if [ ! -f "${LINUX_HDROBJ_DIR}/include/config/auto.conf" ]; then
 647         cp "${LINUX_HDROBJ_DIR}/.config" "${LINUX_HDROBJ_DIR}/include/config/auto.conf"
 648     fi
 649
 650     # Finally clean the object files from the full source tree
 651     make clean
 652
 653     # And regen the modules support files
 654     make modules_prepare "${make_args[@]}"
 655
 656     # On powerpc 32bits this object is required to link modules
 657     if [ "${karch}" = "powerpc" ]; then
 658         if [ "x$(scripts/config -s CONFIG_PPC64)" = "xy" ] && vergte "${kversion}" "5.4"; then
 659             :
 660         else
 661             make arch/powerpc/lib/crtsavres.o "${make_args[@]}"
 662         fi
 663     fi
 664
 665     # On arm64 between 4.13 and 4.15 this object is required to build with ftrace support
 666     if [ "${karch}" = "arm64" ]; then
 667         if [ -f "arch/arm64/kernel/ftrace-mod.S" ]; then
 668             make arch/arm64/kernel/ftrace-mod.o "${make_args[@]}"
 669         fi
 670     fi
 671
 672     # Version specific tasks
 673     case "$ktag" in
 674       Ubuntu*)
 675         # Add Ubuntu ABI number to kernel headers, this is normally done by the packaging code
 676         ABINUM="$(echo "$ktag" | grep -P -o 'Ubuntu-(lts-)?.*-\K\d+(?=\..*)')"
 677         echo "#define UTS_UBUNTU_RELEASE_ABI $ABINUM" >> include/generated/utsrelease.h
 678         echo "#define UTS_UBUNTU_RELEASE_ABI $ABINUM" >> "${LINUX_HDROBJ_DIR}/include/generated/utsrelease.h"
 679         ;;
 680     esac
 681 }
 682
 683
 684 build_modules() {
 685
 686     local kdir="$1"
 687     local outdir="$2"
 688     local kversion
 689
 690     kversion=$(make -C "$LINUX_HDROBJ_DIR" -s kernelversion)
 691
 692     # Try to catch some compatibility problems by turning some
 693     # warnings into errors.
 694     #export KCFLAGS="$KCFLAGS -Wall -Werror"
 695
 696     # Enter lttng-modules source dir
 697     cd "${MODULES_GIT_DIR}"
 698
 699     # kernels 3.10 to 3.10.13 and 3.11 to 3.11.2 introduce a deadlock in the
 700     # timekeeping subsystem. We want those build to fail.
 701     if { vergte "$kversion" "3.10" && verlte "$kversion" "3.10.13"; } || \
 702        { vergte "$kversion" "3.11" && verlte "$kversion" "3.11.2"; }; then
 703
 704         set +e
 705
 706         # Build modules
 707         KERNELDIR="${kdir}" make -j"${NPROC}" V=1 "${make_args[@]}"
 708         ret=$?
 709
 710         set -e
 711
 712         # We expect this build to fail, if it doesn't, fail the job.
 713         if [ "$ret" -eq 0 ]; then
 714             echo "This build should have failed."
 715             exit 1
 716         fi
 717
 718         # We have to publish at least one file or the build will fail
 719         echo "This kernel is broken, there is a deadlock in the timekeeping subsystem." > "${outdir}/BROKEN.txt.ko"
 720
 721         KERNELDIR="${kdir}" make clean
 722
 723     else # Regular build
 724
 725         # Build modules against full kernel sources
 726         KERNELDIR="${kdir}" make -j"${NPROC}" V=1 "${make_args[@]}"
 727
 728         # Install modules to build dir
 729         KERNELDIR="${kdir}" make INSTALL_MOD_PATH="${outdir}" modules_install
 730
 731         # Clean build dir
 732         KERNELDIR="${kdir}" make clean
 733     fi
 734 }
 735
 736
 737 ## MAIN ##
 738
 739 # Use all CPU cores
 740 NPROC=$(nproc)
 741
 742 MODULES_GIT_DIR="${WORKSPACE}/src/lttng-modules"
 743 LINUX_GIT_DIR="${WORKSPACE}/src/linux"
 744
 745 LINUX_OBJ_DIR="${WORKSPACE}/linux"
 746 LINUX_SRCOBJ_DIR="${LINUX_OBJ_DIR}/sources"
 747 LINUX_HDROBJ_DIR="${LINUX_OBJ_DIR}/headers"
 748 LINUX_INSTOBJ_DIR="${LINUX_OBJ_DIR}/install"
 749
 750 MODULES_OUTPUT_KSRC_DIR="${WORKSPACE}/build/lttng-modules-ksrc"
 751 MODULES_OUTPUT_KHDR_DIR="${WORKSPACE}/build/lttng-modules-khdr"
 752
 753 LINUX_GIT_REF_REPO_DIR="$HOME/gitcache/linux-stable.git/"
 754
 755 OBJ_STORE_URL="s3://jenkins"
 756
 757 cd "$WORKSPACE"
 758
 759 # Create build directories
 760 mkdir -p "${LINUX_SRCOBJ_DIR}" "${LINUX_HDROBJ_DIR}" "${LINUX_INSTOBJ_DIR}" "${MODULES_OUTPUT_KSRC_DIR}" "${MODULES_OUTPUT_KHDR_DIR}"
 761
 762 git_clone_modules_sources
 763
 764 # Setup cross compile env if available
 765 if [ "x${cross_arch}" != "x" ]; then
 766
 767     case "$cross_arch" in
 768         "armhf")
 769             karch="arm"
 770             cross_compile="arm-linux-gnueabihf-"
 771             vanilla_config="imx_v6_v7_defconfig"
 772             ubuntu_config="armhf-config.flavour.generic"
 773             ;;
 774
 775         "arm64")
 776             karch="arm64"
 777             cross_compile="aarch64-linux-gnu-"
 778             vanilla_config="defconfig"
 779             ubuntu_config="arm64-config.flavour.generic"
 780             ;;
 781
 782         "powerpc")
 783             karch="powerpc"
 784             cross_compile="powerpc-linux-gnu-"
 785             vanilla_config="ppc44x_defconfig"
 786             ubuntu_config="powerpc-config.flavour.powerpc-smp"
 787             ;;
 788
 789         "ppc64el")
 790             karch="powerpc"
 791             cross_compile="powerpc64le-linux-gnu-"
 792             vanilla_config="pseries_le_defconfig"
 793             ubuntu_config="ppc64el-config.flavour.generic"
 794             ;;
 795
 796         *)
 797             echo "Unsupported cross arch $cross_arch"
 798             exit 1
 799             ;;
 800     esac
 801
 802     # Export variables used by Kbuild for cross compilation
 803     export ARCH="${karch}"
 804     export CROSS_COMPILE="${cross_compile}"
 805
 806 # Set arch specific values if we are not cross compiling
 807 elif [ "x${arch}" != "x" ]; then
 808
 809     case "$arch" in
 810         "i386")
 811             karch="x86"
 812             vanilla_config="allmodconfig"
 813             ubuntu_config="i386-config.flavour.generic"
 814             ;;
 815
 816         "amd64")
 817             karch="x86"
 818             vanilla_config="allmodconfig"
 819             ubuntu_config="amd64-config.flavour.generic"
 820             ;;
 821
 822         "armhf")
 823             karch="arm"
 824             vanilla_config="allmodconfig"
 825             ubuntu_config="armhf-config.flavour.generic"
 826             ;;
 827
 828         "arm64")
 829             karch="arm64"
 830             vanilla_config="allmodconfig"
 831             ubuntu_config="arm64-config.flavour.generic"
 832             ;;
 833
 834         "powerpc")
 835             karch="powerpc"
 836             vanilla_config="allmodconfig"
 837             ubuntu_config="powerpc-config.flavour.powerpc-smp"
 838             ;;
 839
 840         "ppc64el")
 841             karch="powerpc"
 842             vanilla_config="allmodconfig"
 843             ubuntu_config="ppc64el-config.flavour.generic"
 844             ;;
 845
 846         *)
 847             echo "Unsupported arch $arch"
 848             exit 1
 849             ;;
 850     esac
 851 else
 852     echo "No arch or cross_arch specified"
 853     exit 1
 854 fi
 855
 856
 857
 858 # First get the kernel build from the object store, or build it, if it's
 859 # not available.
 860
 861 echo "# Setup endpoint
 862 host_base = obj.internal.efficios.com
 863 host_bucket = obj.internal.efficios.com
 864 bucket_location = us-east-1
 865 use_https = True
 866
 867 # Setup access keys
 868 access_key = jenkins
 869 secret_key = echo123456
 870
 871 # Enable S3 v4 signature APIs
 872 signature_v2 = False" > "$WORKSPACE/.s3cfg"
 873
 874 url_hash="$(echo -n "$kgitrepo" | md5sum | awk '{ print $1 }')"
 875 obj_name="linux.tar.bz2"
 876
 877 if [ "x${cross_arch}" = "x" ]; then
 878         obj_url_prefix="$OBJ_STORE_URL/linux-build/$url_hash/$ktag/platform-${platforms}/$arch/native"
 879 else
 880         obj_url_prefix="$OBJ_STORE_URL/linux-build/$url_hash/$ktag/platform-${platforms}/${cross_arch}"
 881 fi
 882
 883 obj_url="$obj_url_prefix/$obj_name"
 884
 885 set +e
 886 # In s3cmd 2.3, the return code of get when an object does not exist (64)
 887 # is different than in 2.2 (12). The return codes of 's3cmd info' are
 888 # consistent between 2.2 and 2.3.
 889 s3cmd -c "$WORKSPACE/.s3cfg" info "$obj_url"
 890 ret=$?
 891 set -e
 892
 893 case "$ret" in
 894     "0")
 895       s3cmd -c "$WORKSPACE/.s3cfg" get "$obj_url"
 896       extract_archive_obj
 897       ;;
 898
 899     "12")
 900       echo "File not found"
 901
 902       # Build all the things and upload
 903       # then finish the module build...
 904
 905       git_clone_linux_sources
 906       git_export_linux_sources
 907
 908       select_compiler
 909
 910       ## PREPARE FULL LINUX SOURCE TREE
 911       build_linux_kernel
 912
 913       ## EXTRACT DISTRO STYLE KERNEL HEADERS / DEVEL
 914       extract_distro_headers
 915
 916       tar_archive_obj
 917
 918       upload_archive_obj
 919
 920       ;;
 921
 922     *)
 923       echo "Unknown error? Abort"
 924       exit 1
 925       ;;
 926 esac
 927
 928 select_compiler
 929
 930 selected_cc_version="$(echo "${selected_cc}" | cut -d'-' -f2)"
 931 # lttng-modules requires gcc >= 5.1 for aarch64
 932 # @see https://github.com/lttng/lttng-modules/commit/be06402dbdbea2f3394e60ec15c5d3356e2be416
 933 if { verlt "${selected_cc_version}" "5.1"; } && [ "${cross_arch}" = "arm64" ] ; then
 934     echo "Building lltng-modules on aarch64 requires gcc >= 5.1"
 935     exit 0
 936 fi
 937
 938 ## BUILD modules
 939 # Either we downloaded a pre-build kernel or we built it and uploaded
 940 # the archive for future builds.
 941
 942 cd "$WORKSPACE"
 943
 944 # Build modules against full kernel sources
 945 build_modules "${LINUX_SRCOBJ_DIR}" "${MODULES_OUTPUT_KSRC_DIR}"
 946
 947 # Build modules against kernel headers
 948 build_modules "${LINUX_HDROBJ_DIR}" "${MODULES_OUTPUT_KHDR_DIR}"
 949
 950 # Make sure some modules were actually built
 951 tree "${MODULES_OUTPUT_KSRC_DIR}"
 952 if [ "x$(find "${MODULES_OUTPUT_KSRC_DIR}" -name '*.ko*' -printf yes -quit)" != "xyes" ]; then
 953   echo "No modules built!"
 954   exit 1
 955 fi
 956
 957 tree "${MODULES_OUTPUT_KHDR_DIR}"
 958 if [ "x$(find "${MODULES_OUTPUT_KHDR_DIR}" -name '*.ko*' -printf yes -quit)" != "xyes" ]; then
 959   echo "No modules built!"
 960   exit 1
 961 fi
 962
 963 # EOF