Use s3 (minio) for object storage of system_test.

Work toward the use of lava 2.

Keep old normal storage for now.

Signed-off-by: Jonathan Rajotte <jonathan.rajotte-julien@efficios.com>

diff --git a/jobs/system-tests.yaml b/jobs/system-tests.yaml
index 0887e5be2204273434ad35f56328e8b1ac9b119a..67e148cc70d61e0d92c2ed8ad1124b1166a6ddf1 100644 (file)
--- a/jobs/system-tests.yaml
+++ b/jobs/system-tests.yaml
@@ -302,6 +302,18 @@
       - string:
           name: 'BUILD_DEVICE'
           description: 'The target device. (kvm or baremetal)'
+      - string:
+          name: 'S3_STORAGE_KERNEL_FOLDER'
+          description: 'Path to store the Kernel image'
+      - string:
+          name: 'S3_STORAGE_KERNEL_IMAGE'
+          description: 'Path to store the Kernel IMAGE'
+      - string:
+          name: 'S3_STORAGE_LINUX_MODULES'
+          description: 'Path to store the Kernel Modules'
+      - string:
+          name: 'S3_STORAGE_LTTNG_MODULES'
+          description: 'Path to store the LTTng Modules'
 
     properties:
       - build-discarder:

diff --git a/scripts/system-tests/build-kernel.sh b/scripts/system-tests/build-kernel.sh
index 7f0cca768a657614775be16a61d4fc6d7ce250e2..d8def1cccf7289c021ed746fd9b94fd91c72c050 100644 (file)
--- a/scripts/system-tests/build-kernel.sh
+++ b/scripts/system-tests/build-kernel.sh
@@ -29,3 +29,8 @@ $SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.bzImage" "$STORAGE_USER@$STORAGE_HOST
 $SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.config" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_CONFIG"
 $SCP_COMMAND "$DEPLOYDIR/$KERNEL_COMMIT_ID.linux.modules.tar.gz" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_LINUX_MODULES"
 $SCP_COMMAND "$LINUX_PATH/Module.symvers" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_KERNEL_MODULE_SYMVERS"
+
+$S3_COMMAND put "$DEPLOYDIR/$KERNEL_COMMIT_ID.bzImage" "$S3_STORAGE_KERNEL_IMAGE"
+$S3_COMMAND put "$DEPLOYDIR/$KERNEL_COMMIT_ID.config" "$S3_STORAGE_KERNEL_CONFIG"
+$S3_COMMAND put "$DEPLOYDIR/$KERNEL_COMMIT_ID.linux.modules.tar.gz" "$S3_STORAGE_LINUX_MODULES"
+$S3_COMMAND put "$LINUX_PATH/Module.symvers" "$S3_STORAGE_KERNEL_MODULE_SYMVERS"

diff --git a/scripts/system-tests/build-modules.sh b/scripts/system-tests/build-modules.sh
index 04a8c0fa5076bbe89ac1df4bae853c30eb235ae3..8bbeccc541b50f136110c1f4fcf2fd39e13349a5 100644 (file)
--- a/scripts/system-tests/build-modules.sh
+++ b/scripts/system-tests/build-modules.sh
@@ -26,3 +26,4 @@ KERNELDIR="$LINUX_PATH" make -j"$NPROC" --directory="$LTTNG_MODULES_PATH" module
 tar -czf "$DEPLOYDIR/$BUILD_NAME.lttng.modules.tar.gz" -C "$MODULES_INSTALL_FOLDER/" ./
 
 $SCP_COMMAND "$DEPLOYDIR/$BUILD_NAME.lttng.modules.tar.gz" "$STORAGE_USER@$STORAGE_HOST:$STORAGE_LTTNG_MODULES"
+$S3_COMMAND "$DEPLOYDIR/$BUILD_NAME.lttng.modules.tar.gz" "$S3_STORAGE_LTTNG_MODULES"

diff --git a/scripts/system-tests/check-build-needs.sh b/scripts/system-tests/check-build-needs.sh
index 162a7c47af1426a5f5a662749f71fe9b0d7272f4..21acde43cc7a2f1f3fd3eec4ab688f4e784d4653 100644 (file)
--- a/scripts/system-tests/check-build-needs.sh
+++ b/scripts/system-tests/check-build-needs.sh
@@ -27,10 +27,23 @@ if [ $? -ne 0 ]; then
   NEED_MODULES_BUILD=1
 fi
 
+$S3_COMMAND info "$S3_STORAGE_KERNEL_IMAGE"
+if [ $? -ne 0 ]; then
+  NEED_KERNEL_BUILD=1
+  # We need to build the lttng modules if the kernel has changed.
+  NEED_MODULES_BUILD=1
+fi
+
 $SSH_COMMAND "$STORAGE_USER@$STORAGE_HOST" ls "$STORAGE_LTTNG_MODULES"
 if [ $? -ne 0 ]; then
   NEED_MODULES_BUILD=1
 fi
+
+$S3_COMMAND info "$S3_STORAGE_LTTNG_MODULES"
+if [ $? -ne 0 ]; then
+  NEED_MODULES_BUILD=1
+fi
+
 set -e
 
 # We need to fetch the kernel source and lttng-modules to build either the

diff --git a/scripts/system-tests/generate-properties-master.sh b/scripts/system-tests/generate-properties-master.sh
index bb2e25b89502a977e1b5352c630c17b2a0fddcf6..3665835400592b532837cfbe969f464b9aac9f5c 100644 (file)
--- a/scripts/system-tests/generate-properties-master.sh
+++ b/scripts/system-tests/generate-properties-master.sh
@@ -16,6 +16,19 @@
 
 touch properties.txt
 
+echo "# Setup endpoint
+host_base = obj.internal.efficios.com
+host_bucket = obj.internal.efficios.com
+bucket_location = us-east-1
+use_https = True
+
+# Setup access keys
+access_key = jenkins
+secret_key = echo123456
+
+# Enable S3 v4 signature APIs
+signature_v2 = False" > "$WORKSPACE/s3cfg"
+
 LTTNG_CI_PATH="$WORKSPACE/src/lttng-ci"
 echo "LTTNG_CI_PATH=$LTTNG_CI_PATH" >> properties.txt
 KERNEL_COMMIT_ID=$KERNEL_TAG_ID
@@ -37,3 +50,11 @@ echo "STORAGE_KERNEL_FOLDER=$BASE_STORAGE_FOLDER/kernel" >> properties.txt
 echo "STORAGE_KERNEL_IMAGE=$BASE_STORAGE_FOLDER/kernel/$KERNEL_COMMIT_ID.$BUILD_DEVICE.bzImage" >> properties.txt
 echo "STORAGE_LINUX_MODULES=$BASE_STORAGE_FOLDER/modules/linux/$KERNEL_COMMIT_ID.$BUILD_DEVICE.linux.modules.tar.gz" >> properties.txt
 echo "STORAGE_LTTNG_MODULES=$BASE_STORAGE_FOLDER/modules/lttng/$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID.$BUILD_DEVICE.lttng.modules.tar.gz" >> properties.txt
+
+BASE_S3_STORAGE="s3://lava"
+
+echo "BASE_S3_STORAGE=$BASE_S3_STORAGE" >> properties.txt
+echo "S3_STORAGE_KERNEL_FOLDER=$BASE_S3_STORAGE/kernel/" >> properties.txt
+echo "S3_STORAGE_KERNEL_IMAGE=$BASE_S3_STORAGE/kernel/$KERNEL_COMMIT_ID.$BUILD_DEVICE.bzImage" >> properties.txt
+echo "S3_STORAGE_LINUX_MODULES=$BASE_S3_STORAGE/modules/linux/$KERNEL_COMMIT_ID.$BUILD_DEVICE.linux.modules.tar.gz" >> properties.txt
+echo "S3_STORAGE_LTTNG_MODULES=$BASE_S3_STORAGE/modules/lttng/$KERNEL_COMMIT_ID-$LTTNG_MODULES_COMMIT_ID.$BUILD_DEVICE.lttng.modules.tar.gz" >> properties.txt

diff --git a/scripts/system-tests/generate-properties-slave.sh b/scripts/system-tests/generate-properties-slave.sh
index 426e3b50682e113dd6bb3597c6314982c2a65bfa..182d350b7aac57151306b9a77a3d655a195f7426 100644 (file)
--- a/scripts/system-tests/generate-properties-slave.sh
+++ b/scripts/system-tests/generate-properties-slave.sh
@@ -16,6 +16,19 @@
 
 touch properties.txt
 
+echo "# Setup endpoint
+host_base = obj.internal.efficios.com
+host_bucket = obj.internal.efficios.com
+bucket_location = us-east-1
+use_https = True
+
+# Setup access keys
+access_key = jenkins
+secret_key = echo123456
+
+# Enable S3 v4 signature APIs
+signature_v2 = False" > "$WORKSPACE/s3cfg"
+
 # Use all CPU cores
 NPROC=$(nproc)
 echo "NPROC=$NPROC" >> properties.txt
@@ -42,9 +55,12 @@ echo "BUILD_DEVICE=$BUILD_DEVICE" >> properties.txt
 
 echo "STORAGE_KERNEL_MODULE_SYMVERS=$STORAGE_KERNEL_FOLDER/symvers/$KERNEL_COMMIT_ID.$BUILD_DEVICE.symvers" >>properties.txt
 echo "STORAGE_KERNEL_CONFIG=$STORAGE_KERNEL_FOLDER/config/$KERNEL_COMMIT_ID.$BUILD_DEVICE.config" >> properties.txt
+echo "S3_STORAGE_KERNEL_MODULE_SYMVERS=$S3_STORAGE_KERNEL_FOLDER/symvers/$KERNEL_COMMIT_ID.$BUILD_DEVICE.symvers" >>properties.txt
+echo "S3_STORAGE_KERNEL_CONFIG=$S3_STORAGE_KERNEL_FOLDER/config/$KERNEL_COMMIT_ID.$BUILD_DEVICE.config" >> properties.txt
 
 echo "STORAGE_HOST=storage.internal.efficios.com" >> properties.txt
 echo "STORAGE_USER=jenkins-lava" >> properties.txt
 
 echo SSH_COMMAND="ssh -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt
 echo SCP_COMMAND="scp -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt
+echo S3_COMMAND="s3cmd -c ${WORKSPACE}/s3cfg" >> properties.txt

diff --git a/scripts/system-tests/inject-ssh-commands.sh b/scripts/system-tests/inject-ssh-commands.sh
index 74e4dde4caf9ffa92f6c3cadc36498a9fb73eb71..8f3598c41378ebb660f1629bbf4cf479b25708a5 100644 (file)
--- a/scripts/system-tests/inject-ssh-commands.sh
+++ b/scripts/system-tests/inject-ssh-commands.sh
@@ -16,3 +16,4 @@
 
 echo SSH_COMMAND="ssh -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt
 echo SCP_COMMAND="scp -oStrictHostKeyChecking=no -i $identity_file" >> properties.txt
+echo S3_COMMAND="s3cmd -c ${WORKSPACE}/s3cfg" >> properties.txt