pipelines/images/imagebuild.sh

   1 #!/usr/bin/bash -eux
   2
   3 CLEANUP=()
   4
   5 function cleanup {
   6     set +e
   7     for (( index=${#CLEANUP[@]}-1 ; index >= 0 ; index-- )) ;do
   8         ${CLEANUP[$index]}
   9     done
  10     CLEANUP=()
  11     set -e
  12 }
  13
  14 function fail {
  15     CODE="${1:-1}"
  16     REASON="${2:-Unknown reason}"
  17     cleanup
  18     echo "${REASON}" >&2
  19     exit "${CODE}"
  20 }
  21
  22 trap cleanup EXIT TERM INT
  23
  24 env
  25
  26 REQUIRED_VARIABLES=(
  27     OS              # OS name
  28     RELEASE         # OS release
  29     ARCH            # The image architecture
  30     IMAGE_TYPE      # The image type to create
  31     VARIANT         # The variant of the base image to use
  32     PROFILE         # The ansible group to apply to the new image
  33     GIT_BRANCH      # The git branch of the automation repo to checkout
  34     GIT_URL         # The git URL of the automation repo to checkout
  35     LXD_CLIENT_CERT # Path to LXD client certificate
  36     LXD_CLIENT_KEY  # Path to LXD client certificate key
  37     SSH_PRIVATE_KEY # Path to SSH private key
  38     TEST            # 'true' to test launching published image
  39 )
  40 MISSING_VARS=0
  41 for var in "${REQUIRED_VARIABLES[@]}" ; do
  42     if [ ! -v "$var" ] ; then
  43         MISSING_VARS=1
  44         echo "Missing required variable: '${var}'" >&2
  45     fi
  46 done
  47 if [[ ! "${MISSING_VARS}" == "0" ]] ; then
  48     fail 1 "Missing required variables"
  49 fi
  50
  51 # Default optional variables
  52 INSTANCE_START_TIMEOUT="${INSTANCE_START_TIMEOUT:-120}"
  53 NETWORK_SLEEP="${NETWORK_SLEEP:-15}"
  54
  55 # Dependencies
  56 apt-get -y install lxd-client ansible jq
  57
  58 # Configuration
  59 mkdir -p ~/.config/lxc
  60 cp "${LXD_CLIENT_CERT}" ~/.config/lxc/client.crt
  61 cp "${LXD_CLIENT_KEY}" ~/.config/lxc/client.key
  62 CLEANUP+=(
  63     "rm -f ${HOME}/.config/lxc/client.crt"
  64     "rm -f ${HOME}/.config/lxc/client.key"
  65 )
  66 lxc remote add ci --accept-certificate --auth-type tls "${LXD_HOST}"
  67 lxc remote switch ci
  68
  69 # Clone lttng-ci
  70 git clone -b "${GIT_BRANCH}" "${GIT_URL}" ci
  71 cd ci/automation/ansible || exit 1
  72
  73 SOURCE_IMAGE_NAME="${OS}/${RELEASE}/${VARIANT}/${ARCH}"
  74 # Include IMAGE_TYPE since an alias may only be defined once even if the
  75 # type of the image differs
  76 TARGET_IMAGE_NAME="${OS}/${RELEASE}/${VARIANT}/${ARCH}/${PROFILE}/${IMAGE_TYPE}"
  77 INSTANCE_NAME=''
  78 # Try from local cache
  79 VM_ARG=()
  80 if [ "${IMAGE_TYPE}" == "vm" ] ; then
  81     VM_ARG=("--vm")
  82 fi
  83
  84 set +e
  85 # Test
  86 # It's possible that concurrent image creation when running parallel jobs causes
  87 # an error during the launch:
  88 #   Error: Failed instance creation: UNIQUE constraint failed: images.project_id, images.fingerprint
  89 # C.f. https://github.com/canonical/lxd/issues/11636
  90 #
  91 TRIES_MAX=3
  92 TRIES=0
  93 while [[ "${TRIES}" -lt "${TRIES_MAX}" ]] ; do
  94     if ! INSTANCE_NAME=$(lxc -q launch -e "${VM_ARG[@]}" -p default -p "${LXD_INSTANCE_PROFILE}" "${SOURCE_IMAGE_NAME}/${IMAGE_TYPE}") ; then
  95         # Try from images
  96         if ! INSTANCE_NAME=$(lxc -q launch -e "${VM_ARG[@]}" -p default -p "${LXD_INSTANCE_PROFILE}" images:"${SOURCE_IMAGE_NAME}") ; then
  97             TRIES=$((TRIES + 1))
  98             echo "Failed to deployed ephemereal instance attempt ${TRIES}/${TRIES_MAX}"
  99             if [[ "${TRIES}" -lt  "${TRIES_MAX}" ]] ; then
 100                 continue
 101             fi
 102             fail 1 "Failed to deploy ephemereal instance"
 103         else
 104             break
 105         fi
 106     else
 107         break
 108     fi
 109 done
 110 INSTANCE_NAME="$(echo "${INSTANCE_NAME}" | cut -d ':' -f 2 | tr -d ' ')"
 111 set -e
 112
 113 CLEANUP+=(
 114     "lxc stop -f ${INSTANCE_NAME}"
 115 )
 116
 117 # VMs may take more time to start, wait until instance is running
 118 TIME_REMAINING="${INSTANCE_START_TIMEOUT}"
 119 while true ; do
 120     set +e
 121     INSTANCE_STATUS=$(lxc exec "${INSTANCE_NAME}" hostname)
 122     set -e
 123     if [[ "${INSTANCE_STATUS}" == "${INSTANCE_NAME}" ]] ; then
 124         break
 125     fi
 126     sleep 1
 127     TIME_REMAINING=$((TIME_REMAINING - 1))
 128     if [ "${TIME_REMAINING}" -lt "0" ] ; then
 129         fail 1 "Timed out waiting for instance to become available via 'lxc exec'"
 130     fi
 131 done
 132
 133 # Wait for cloud-init to finish
 134 if [[ "${VARIANT}" == "cloud" ]] ; then
 135     # It's possible for cloud-init to fail, but to still be able to continue.
 136     # Eg., a profile asks for netplan.io on a system that doesn't have that
 137     # package available.
 138     lxc exec "${INSTANCE_NAME}" -- cloud-init status -w || true
 139 fi
 140
 141 # Wait for instance to have an ip address (@TODO: is there a better approach?)
 142 sleep "${NETWORK_SLEEP}"
 143
 144 # @TODO: Handle case when iputils2 is not installed
 145 INSTANCE_IP=''
 146 POTENTIAL_INTERFACES=(eth0 enp5s0)
 147 lxc exec "${INSTANCE_NAME}" -- ip a
 148 set +e
 149 for interface in "${POTENTIAL_INTERFACES[@]}" ; do
 150     if ! DEV_INFO="$(lxc exec "${INSTANCE_NAME}" -- ip a show dev "${interface}")" ; then
 151         continue
 152     fi
 153     INSTANCE_IP="$(echo "${DEV_INFO}" | grep -Eo 'inet [^ ]* ' | cut -d' ' -f2 | cut -d'/' -f1)"
 154     if [[ "${INSTANCE_IP}" != "" ]] ; then
 155         break
 156     fi
 157 done
 158 set -e
 159 if [[ "${INSTANCE_IP}" == "" ]] ; then
 160     fail 1 "Failed to determine instance IP address"
 161 fi
 162
 163 ssh-keyscan "${INSTANCE_IP}" >> ~/.ssh/known_hosts2
 164 #lxc exec "${INSTANCE_NAME}" -- bash -c 'for i in /etc/ssh/ssh_host_*_key ; do ssh-keygen -l -f "$i" ; done' >> "${HOME}/.ssh/known_hosts"
 165 CLEANUP+=(
 166     "rm -f ${HOME}/.ssh/known_hosts2"
 167 )
 168 cp "${SSH_PRIVATE_KEY}" ~/.ssh/id_rsa
 169 ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub
 170 CLEANUP+=(
 171     "rm -f ${HOME}/.ssh/id_rsa.pub"
 172     "rm -f ${HOME}/.ssh/id_rsa"
 173 )
 174 lxc file push ~/.ssh/id_rsa.pub "ci:${INSTANCE_NAME}/root/.ssh/authorized_keys2"
 175
 176 # Confirm working SSH connection
 177 if ! ssh "${INSTANCE_IP}" hostname ; then
 178     fail 1 "Unable to reach ephemereal instance over SSH"
 179 fi
 180
 181 # Run playbook
 182 cat > fake-inventory <<EOF
 183 [${PROFILE/-/_}]
 184 ${INSTANCE_IP}
 185 EOF
 186 CLEANUP+=(
 187     "rm -f $(pwd)/fake-inventory"
 188 )
 189
 190 LANG=C ANSIBLE_STRATEGY=linear ansible-playbook site.yml \
 191     -e '{"compilers_legacy_install": false, "jenkins_user": false, "lttng_modules_checkout_repo": false}' \
 192     -l "${INSTANCE_IP}" -i fake-inventory
 193
 194 # Cleanup instance side
 195 LANG=C ANSIBLE_STRATEGY=linear ansible-playbook \
 196        playbooks/post-imagebuild-clean.yml \
 197        -l "${INSTANCE_IP}" -i fake-inventory
 198
 199 # Publish
 200 if FINGERPRINT=$(lxc publish "${INSTANCE_NAME}" -f 2>&1 | grep -E -o '[A-Fa-f0-9]{64}') ; then
 201     echo "Published instance with fingerprint '${FINGERPRINT}'"
 202 else
 203     fail 1 "No fingerprint for published instance"
 204 fi
 205
 206 TRIES=0
 207
 208 if [[ "${TEST}" == "true" ]] ; then
 209     set +e
 210     while [[ "${TRIES}" -lt "${TRIES_MAX}" ]] ; do
 211         if ! INSTANCE_NAME=$(lxc -q launch -e "${VM_ARG[@]}" -p default -p "${LXD_INSTANCE_PROFILE}" "${FINGERPRINT}")  ; then
 212             TRIES=$((TRIES + 1))
 213             echo "Failed to launch instance try ${TRIES}/${TRIES_MAX}"
 214             if [[ "${TRIES}" -lt "${TRIES_MAX}" ]] ; then
 215                 sleep $((1 + RANDOM % 10))
 216                 continue
 217             fi
 218             fail 1 "Failed to launch an instance using newly published image '${FINGERPRINT}'"
 219         else
 220             INSTANCE_NAME="$(echo "${INSTANCE_NAME}" | cut -d':' -f2 | tr -d ' ')"
 221             CLEANUP+=(
 222                 "lxc stop -f ${INSTANCE_NAME}"
 223             )
 224             break
 225         fi
 226     done
 227     set -e
 228 fi
 229
 230 lxc image alias delete "${TARGET_IMAGE_NAME}" || true
 231 lxc image alias create "${TARGET_IMAGE_NAME}" "${FINGERPRINT}"