pipelines/images/distrobuild.sh

   1 #!/usr/bin/bash -eux
   2
   3 CLEANUP=()
   4
   5 function cleanup {
   6     set +e
   7     for (( index=${#CLEANUP[@]}-1 ; index >= 0 ; index-- )) ;do
   8         ${CLEANUP[$index]}
   9     done
  10     CLEANUP=()
  11     set -e
  12 }
  13
  14 function fail {
  15     CODE="${1:-1}"
  16     REASON="${2:-Unknown reason}"
  17     cleanup
  18     echo "${REASON}" >&2
  19     exit "${CODE}"
  20 }
  21
  22 trap cleanup EXIT TERM INT
  23
  24 env
  25
  26 REQUIRED_VARIABLES=(
  27     OS
  28     RELEASE
  29     ARCH
  30     IMAGE_TYPE
  31     VARIANT
  32     GIT_BRANCH
  33     GIT_URL
  34     LXD_CLIENT_CERT
  35     LXD_CLIENT_KEY
  36     TEST
  37     DISTROBUILDER_GIT_URL
  38     DISTROBUILDER_GIT_BRANCH
  39     LXC_CI_GIT_URL
  40     LXC_CI_GIT_BRANCH
  41     GO_VERSION
  42 )
  43 MISSING_VARS=0
  44 for var in "${REQUIRED_VARIABLES[@]}" ; do
  45     if [ ! -v "$var" ] ; then
  46         MISSING_VARS=1
  47         echo "Missing required variable: '${var}'" >&2
  48     fi
  49 done
  50 if [[ ! "${MISSING_VARS}" == "0" ]] ; then
  51     fail 1 "Missing required variables"
  52 fi
  53
  54 # Optional variables
  55 INSTANCE_START_TIMEOUT="${INSTANCE_START_TIMEOUT:-60}"
  56 VM_ARG=()
  57
  58 # Install lxd-client
  59 apt-get update
  60 apt-get install -y lxd-client
  61 mkdir -p ~/.config/lxc
  62 cp "${LXD_CLIENT_CERT}" ~/.config/lxc/client.crt
  63 cp "${LXD_CLIENT_KEY}" ~/.config/lxc/client.key
  64 CLEANUP+=(
  65     "rm -f ${HOME}/.config/lxc/client.crt"
  66     "rm -f ${HOME}/.config/lxc/client.key"
  67 )
  68 lxc remote add ci --accept-certificate --auth-type tls "${LXD_HOST}"
  69 lxc remote switch ci
  70
  71 # Exit gracefully if the lxc images: provides the base image
  72 IMAGE_NAME="${OS}/${RELEASE}/${VARIANT}/${ARCH}"
  73 TYPE_FILTER='type=container'
  74 if [[ "${IMAGE_TYPE}" == "vm" ]] ; then
  75     TYPE_FILTER='type=virtual-machine'
  76 fi
  77 if [[ "$(lxc image list -f csv images:"${IMAGE_NAME}" -- "${TYPE_FILTER}" | wc -l)" != "0" ]] ; then
  78     echo "Image '${IMAGE_NAME}' provided by 'images:' remote"
  79     exit 0
  80 fi
  81
  82 # Get go
  83 apt-get install -y wget
  84 wget "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" -O - | tar -C /usr/local -xzf -
  85 export PATH="${PATH}:/usr/local/go/bin"
  86
  87 # Install distrobuilder
  88 apt-get install -y debootstrap rsync gpg squashfs-tools git \
  89         btrfs-progs dosfstools qemu-utils gdisk
  90 cd "${WORKSPACE}"
  91 git clone --branch="${DISTROBUILDER_GIT_BRANCH}" "${DISTROBUILDER_GIT_URL}" distrobuilder
  92 cd distrobuilder
  93 make
  94 PATH="${PATH}:${HOME}/go/bin"
  95
  96 # Get CI repo
  97 cd "${WORKSPACE}"
  98 git clone --branch="${GIT_BRANCH}" "${GIT_URL}" ci
  99
 100 # Get the LXC CI repo
 101 cd "${WORKSPACE}"
 102 git clone --branch="${LXC_CI_GIT_BRANCH}" "${LXC_CI_GIT_URL}" lxc-ci
 103
 104 IMAGE_DIRS=(
 105     "${WORKSPACE}/ci/automation/images"
 106     "${WORKSPACE}/lxc-ci/images"
 107 )
 108 EXTENSIONS=(
 109     'yml'
 110     'yaml'
 111 )
 112 IMAGE_FILE=''
 113 for IMAGE_DIR in "${IMAGE_DIRS[@]}" ; do
 114     for EXTENSION in "${EXTENSIONS[@]}" ; do
 115         if [ -f "${IMAGE_DIR}/${OS}-${RELEASE}.${EXTENSION}" ] ; then
 116             IMAGE_FILE="${IMAGE_DIR}/${OS}-${RELEASE}.${EXTENSION}"
 117             break 2;
 118         fi
 119     done
 120     for EXTENSION in "${EXTENSIONS[@]}" ; do
 121         if [ -f "${IMAGE_DIR}/${OS}.${EXTENSION}" ] ; then
 122             IMAGE_FILE="${IMAGE_DIR}/${OS}.${EXTENSION}"
 123             break 2;
 124         fi
 125     done
 126 done
 127
 128 if [[ "${IMAGE_FILE}" == "" ]] ; then
 129     fail 1 "Unable to find image file for '${OS}' in ${IMAGE_DIRS[@]}"
 130 fi
 131
 132 if grep -q -E 'XX[A-Za-z0-9_]+XX' "${IMAGE_FILE}" ; then
 133     while read -r VAR ; do
 134         echo "${VAR}"
 135         SHELLVAR=$(echo "${VAR}" | sed 's/^XX//g' | sed 's/XX$//g')
 136         set +x
 137         sed -i "s/${VAR}/${!SHELLVAR:-VARIABLENOTFOUND}/g" "${IMAGE_FILE}"
 138         set -x
 139     done < <(grep -E -o 'XX[A-Za-z0-9_]+XX' "${IMAGE_FILE}")
 140 fi
 141
 142 DISTROBUILDER_ARGS=(
 143     distrobuilder
 144     build-incus
 145 )
 146 if [[ "${IMAGE_TYPE}" == "vm" ]] ; then
 147     DISTROBUILDER_ARGS+=('--vm')
 148     VM_ARG=('--vm')
 149 fi
 150
 151 # This could be quite large, and /tmp may be a tmpfs backed
 152 # by memory, so instead make it relative to the workspace directory
 153 BUILD_DIR=$(mktemp -d -p "${WORKSPACE}")
 154 CLEANUP+=(
 155     "rm -rf ${BUILD_DIR}"
 156 )
 157 DISTROBUILDER_ARGS+=(
 158     "${IMAGE_FILE}"
 159     "${BUILD_DIR}"
 160     '-o'
 161     "image.architecture=${ARCH}"
 162     '-o'
 163     "image.variant=${VARIANT}"
 164     '-o'
 165     "image.release=${RELEASE}"
 166     '-o'
 167     "image.serial=$(date -u +%Y%m%dT%H:%M:%S%z)"
 168 )
 169
 170 # Run the build
 171 ${DISTROBUILDER_ARGS[@]}
 172
 173 # Import
 174 # As 'distrobuilder --import-into-incus=alias' doesn't work since it only
 175 # connects to the local unix socket, and the remote instance cannot be specified
 176 # at this time.
 177 ROOTFS="${BUILD_DIR}/rootfs.squashfs"
 178 if [[ "${IMAGE_TYPE}" == "vm" ]] ; then
 179     ROOTFS="${BUILD_DIR}/disk.qcow2"
 180 fi
 181
 182 # Work-around for lxd not using qemu-system-i386: set the architecture to x86_64
 183 # which will use qemu-system-x86_64 and still run 32bit userspace/kernels fine.
 184 if [[ "${ARCH}" == "i386" ]] ; then
 185     TMP_DIR=$(mktemp -d)
 186     pushd "${TMP_DIR}"
 187     tar -xf "${BUILD_DIR}/incus.tar.xz"
 188     sed -i 's/architecture: i386/architecture: x86_64/' metadata.yaml
 189     tar -cf "${BUILD_DIR}/incus.tar.xz" ./*
 190     popd
 191     rm -rf "${TMP_DIR}"
 192 fi
 193
 194 # When using `lxc image import` two images cannot have the same alias -
 195 # only the last image imported will keep the alias. Therefore, the
 196 # image type is appended as part of the alias.
 197 IMAGE_NAME="${IMAGE_NAME}/${IMAGE_TYPE}"
 198 lxc image import "${BUILD_DIR}/incus.tar.xz" "${ROOTFS}" --alias="${IMAGE_NAME}" ci:
 199
 200 if [[ "${TEST}" == "true" ]] ; then
 201     set +e
 202     INSTANCE_NAME=''
 203     if INSTANCE_NAME="$(lxc -q launch -e ${VM_ARG[@]} -p default -p "${LXD_INSTANCE_PROFILE}" "${IMAGE_NAME}")" ; then
 204         INSTANCE_NAME="$(echo "${INSTANCE_NAME}" | cut -d':' -f2 | tr -d ' ')"
 205         CLEANUP+=(
 206             "lxc stop ${INSTANCE_NAME}"
 207         )
 208     else
 209         fail 1 "Failed to launch instance using image '${IMAGE_NAME}'"
 210     fi
 211     TIME_REMAINING="${INSTANCE_START_TIMEOUT}"
 212     INSTANCE_STATUS=''
 213     while true ; do
 214         INSTANCE_STATUS="$(lxc exec "${INSTANCE_NAME}" hostname)"
 215         if [[ "${INSTANCE_STATUS}" == "${INSTANCE_NAME}" ]] ; then
 216             break
 217         fi
 218         sleep 1
 219         TIME_REMAINING=$((TIME_REMAINING - 1))
 220         if [ "${TIME_REMAINING}" -lt "0" ] ; then
 221             fail 1 "Timed out waiting for instance to become available via 'lxc exec'"
 222         fi
 223     done
 224     set -e
 225 fi