pipelines/images/distrobuild.sh

   1 #!/usr/bin/bash -eux
   2
   3 CLEANUP=()
   4
   5 function cleanup {
   6     set +e
   7     for (( index=${#CLEANUP[@]}-1 ; index >= 0 ; index-- )) ;do
   8         ${CLEANUP[$index]}
   9     done
  10     CLEANUP=()
  11     set -e
  12 }
  13
  14 function fail {
  15     CODE="${1:-1}"
  16     REASON="${2:-Unknown reason}"
  17     cleanup
  18     echo "${REASON}" >&2
  19     exit "${CODE}"
  20 }
  21
  22 trap cleanup EXIT TERM INT
  23
  24 env
  25
  26 REQUIRED_VARIABLES=(
  27     OS
  28     RELEASE
  29     ARCH
  30     IMAGE_TYPE
  31     VARIANT
  32     GIT_BRANCH
  33     GIT_URL
  34     LXD_CLIENT_CERT
  35     LXD_CLIENT_KEY
  36     TEST
  37     DISTROBUILDER_GIT_URL
  38     DISTROBUILDER_GIT_BRANCH
  39     LXC_CI_GIT_URL
  40     LXC_CI_GIT_BRANCH
  41     GO_VERSION
  42 )
  43 MISSING_VARS=0
  44 for var in "${REQUIRED_VARIABLES[@]}" ; do
  45     if [ ! -v "$var" ] ; then
  46         MISSING_VARS=1
  47         echo "Missing required variable: '${var}'" >&2
  48     fi
  49 done
  50 if [[ ! "${MISSING_VARS}" == "0" ]] ; then
  51     fail 1 "Missing required variables"
  52 fi
  53
  54 # Optional variables
  55 INSTANCE_START_TIMEOUT="${INSTANCE_START_TIMEOUT:-30}"
  56 VM_ARG=()
  57
  58 # Install lxd-client
  59 apt-get update
  60 apt-get install -y lxd-client
  61 mkdir -p ~/.config/lxc
  62 cp "${LXD_CLIENT_CERT}" ~/.config/lxc/client.crt
  63 cp "${LXD_CLIENT_KEY}" ~/.config/lxc/client.key
  64 CLEANUP+=(
  65     "rm -f ${HOME}/.config/lxc/client.crt"
  66     "rm -f ${HOME}/.config/lxc/client.key"
  67 )
  68 lxc remote add ci --accept-certificate --auth-type tls "${LXD_HOST}"
  69 lxc remote switch ci
  70
  71 # Exit gracefully if the lxc images: provides the base image
  72 IMAGE_NAME="${OS}/${RELEASE}/${VARIANT}/${ARCH}"
  73 TYPE_FILTER='type=container'
  74 if [[ "${IMAGE_TYPE}" == "vm" ]] ; then
  75     TYPE_FILTER='type=virtual-machine'
  76 fi
  77 if [[ "$(lxc image list -f csv images:"${IMAGE_NAME}" -- "${TYPE_FILTER}" | wc -l)" != "0" ]] ; then
  78     echo "Image '${IMAGE_NAME}' provided by 'images:' remote"
  79     exit 0
  80 fi
  81
  82 # Get go
  83 apt-get install -y wget
  84 wget "https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz" -O - | tar -C /usr/local -xzf -
  85 export PATH="${PATH}:/usr/local/go/bin"
  86
  87 # Install distrobuilder
  88 apt-get install -y debootstrap rsync gpg squashfs-tools git \
  89         btrfs-progs dosfstools qemu-utils gdisk
  90 cd "${WORKSPACE}"
  91 git clone --branch="${DISTROBUILDER_GIT_BRANCH}" "${DISTROBUILDER_GIT_URL}" distrobuilder
  92 cd distrobuilder
  93 make
  94 PATH="${PATH}:${HOME}/go/bin"
  95
  96 # Get CI repo
  97 cd "${WORKSPACE}"
  98 git clone --branch="${GIT_BRANCH}" "${GIT_URL}" ci
  99
 100 # Get the LXC CI repo
 101 cd "${WORKSPACE}"
 102 git clone --branch="${LXC_CI_GIT_BRANCH}" "${LXC_CI_GIT_URL}" lxc-ci
 103
 104 IMAGE_DIRS=(
 105     "${WORKSPACE}/ci/automation/images"
 106     "${WORKSPACE}/lxc-ci/images"
 107 )
 108 EXTENSIONS=(
 109     'yml'
 110     'yaml'
 111 )
 112 IMAGE_FILE=''
 113 for IMAGE_DIR in "${IMAGE_DIRS[@]}" ; do
 114     for EXTENSION in "${EXTENSIONS[@]}" ; do
 115         if [ -f "${IMAGE_DIR}/${OS}-${RELEASE}.${EXTENSION}" ] ; then
 116             IMAGE_FILE="${IMAGE_DIR}/${OS}-${RELEASE}.${EXTENSION}"
 117             break 2;
 118         fi
 119     done
 120     for EXTENSION in "${EXTENSIONS[@]}" ; do
 121         if [ -f "${IMAGE_DIR}/${OS}.${EXTENSION}" ] ; then
 122             IMAGE_FILE="${IMAGE_DIR}/${OS}.${EXTENSION}"
 123             break 2;
 124         fi
 125     done
 126 done
 127
 128 if [[ "${IMAGE_FILE}" == "" ]] ; then
 129     fail 1 "Unable to find image file for '${OS}' in ${IMAGE_DIRS[@]}"
 130 fi
 131
 132 DISTROBUILDER_ARGS=(
 133     distrobuilder
 134     build-incus
 135 )
 136 if [[ "${IMAGE_TYPE}" == "vm" ]] ; then
 137     DISTROBUILDER_ARGS+=('--vm')
 138     VM_ARG=('--vm')
 139 fi
 140
 141 # This could be quite large, and /tmp may be a tmpfs backed
 142 # by memory, so instead make it relative to the workspace directory
 143 BUILD_DIR=$(mktemp -d -p "${WORKSPACE}")
 144 CLEANUP+=(
 145     "rm -rf ${BUILD_DIR}"
 146 )
 147 DISTROBUILDER_ARGS+=(
 148     "${IMAGE_FILE}"
 149     "${BUILD_DIR}"
 150     '-o'
 151     "image.architecture=${ARCH}"
 152     '-o'
 153     "image.variant=${VARIANT}"
 154     '-o'
 155     "image.release=${RELEASE}"
 156     '-o'
 157     "image.serial=$(date -u +%Y%m%dT%H:%M:%S%z)"
 158 )
 159
 160 # Run the build
 161 ${DISTROBUILDER_ARGS[@]}
 162
 163 # Import
 164 # As 'distrobuilder --import-into-incus=alias' doesn't work since it only
 165 # connects to the local unix socket, and the remote instance cannot be specified
 166 # at this time.
 167 ROOTFS="${BUILD_DIR}/rootfs.squashfs"
 168 if [[ "${IMAGE_TYPE}" == "vm" ]] ; then
 169     ROOTFS="${BUILD_DIR}/disk.qcow2"
 170 fi
 171
 172 # Work-around for lxd not using qemu-system-i386: set the architecture to x86_64
 173 # which will use qemu-system-x86_64 and still run 32bit userspace/kernels fine.
 174 if [[ "${ARCH}" == "i386" ]] ; then
 175     TMP_DIR=$(mktemp -d)
 176     pushd "${TMP_DIR}"
 177     tar -xf "${BUILD_DIR}/incus.tar.xz"
 178     sed -i 's/architecture: i386/architecture: x86_64/' metadata.yaml
 179     tar -cf "${BUILD_DIR}/incus.tar.xz" ./*
 180     popd
 181     rm -rf "${TMP_DIR}"
 182 fi
 183
 184 lxc image import "${BUILD_DIR}/incus.tar.xz" "${ROOTFS}" --alias="${IMAGE_NAME}" ci:
 185
 186 if [[ "${TEST}" == "true" ]] ; then
 187     set +e
 188     INSTANCE_NAME=''
 189     if INSTANCE_NAME="$(lxc -q launch -e ${VM_ARG[@]} -p default -p "${LXD_INSTANCE_PROFILE}" "${IMAGE_NAME}")" ; then
 190         INSTANCE_NAME="$(echo "${INSTANCE_NAME}" | cut -d':' -f2 | tr -d ' ')"
 191         CLEANUP+=(
 192             "lxc stop ${INSTANCE_NAME}"
 193         )
 194     else
 195         fail 1 "Failed to launch instance using image '${IMAGE_NAME}'"
 196     fi
 197     TIME_REMAINING="${INSTANCE_START_TIMEOUT}"
 198     INSTANCE_STATUS=''
 199     while true ; do
 200         INSTANCE_STATUS="$(lxc exec "${INSTANCE_NAME}" hostname)"
 201         if [[ "${INSTANCE_STATUS}" == "${INSTANCE_NAME}" ]] ; then
 202             break
 203         fi
 204         sleep 1
 205         TIME_REMAINING=$((TIME_REMAINING - 1))
 206         if [ "${TIME_REMAINING}" -lt "0" ] ; then
 207             fail 1 "Timed out waiting for instance to become available via 'lxc exec'"
 208         fi
 209     done
 210     set -e
 211 fi