From 2a497d611acf2b0a3a87b37136ec16b935496097 Mon Sep 17 00:00:00 2001
From: Kevin Carter <kevin.carter@rackspace.com>
Date: Mon, 31 Mar 2025 16:10:42 -0500
Subject: [PATCH] feat: add per-pr smoke tests

This change will build a simple three node cloud environment on-top of
openstack flex in dfw. The job will build, evaluate, and return the
state of the build upon completion. The hyperconverged lab script is
used to run the base test.

The new job will also ensure that it cleans up resources built
no matter the state of the build.

Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
---
 .github/workflows/smoke-deploy-lab.yaml | 38 +++++++++++++++++++++++++
 scripts/hyperconverged-lab.sh           | 23 +++++++++------
 2 files changed, 53 insertions(+), 8 deletions(-)
 create mode 100644 .github/workflows/smoke-deploy-lab.yaml

diff --git a/.github/workflows/smoke-deploy-lab.yaml b/.github/workflows/smoke-deploy-lab.yaml
new file mode 100644
index 00000000..d4e6308d
--- /dev/null
+++ b/.github/workflows/smoke-deploy-lab.yaml
@@ -0,0 +1,38 @@
+name: Running hyperconverged smoke tests
+
+on:
+  pull_request:
+    paths:
+      - ansible/**
+      - base-kustomize/**
+      - base-helm-configs/**
+      - bin/**
+      - scripts/**
+
+env:
+  HYPERCONVERGED_DEV: "true"
+  GATEWAY_DOMAIN: cloud.local
+  ACME_EMAIL: cloud@example.local
+  OS_IMAGE: "Ubuntu 24.04"
+  SSH_USERNAME: "ubuntu"
+
+jobs:
+  smoke-deploy:
+    runs-on: self-hosted
+    container:
+      image: localhost:5000/genestack-ci:latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Dynamically set MY_DATE environment variable
+        run: echo "LAB_NAME_PREFIX=smoke-$(date +%s)" >> $GITHUB_ENV
+
+      - name: Run deployment script
+        run: |
+          eval "$(ssh-agent -s)"
+          scripts/hyperconverged-lab.sh
+
+      - name: Cleanup the lab
+        if: always()
+        run: scripts/hyperconverged-lab-uninstall.sh
diff --git a/scripts/hyperconverged-lab.sh b/scripts/hyperconverged-lab.sh
index 54d509e7..9c13fbb7 100755
--- a/scripts/hyperconverged-lab.sh
+++ b/scripts/hyperconverged-lab.sh
@@ -246,6 +246,11 @@ if ! COMPUTE_2_PORT=$(openstack port show ${LAB_NAME_PREFIX}-2-compute-port -f v
   )
 fi
 
+if [ ! -d "~/.ssh" ]; then
+  echo "Creating the SSH directory"
+  mkdir -p ~/.ssh
+  chmod 700 ~/.ssh
+fi
 if ! openstack keypair show ${LAB_NAME_PREFIX}-key 2> /dev/null; then
     if [ ! -f ~/.ssh/${LAB_NAME_PREFIX}-key.pem ]; then
       openstack keypair create ${LAB_NAME_PREFIX}-key > ~/.ssh/${LAB_NAME_PREFIX}-key.pem
@@ -290,11 +295,11 @@ fi
 
 echo "Waiting for the jump host to be ready"
 COUNT=0
-while ! ssh -o StrictHostKeyChecking=no -o ConnectTimeout=2 -o ConnectionAttempts=3 -o UserKnownHostsFile=/dev/null -q ${SSH_USERNAME}@${JUMP_HOST_VIP} exit; do
+while ! ssh -o ConnectTimeout=2 -o ConnectionAttempts=3 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -q ${SSH_USERNAME}@${JUMP_HOST_VIP} exit; do
    sleep 2
    echo "SSH is not ready, Trying again..."
    COUNT=$((COUNT+1))
-   if [ $COUNT -gt 30 ]; then
+   if [ $COUNT -gt 60 ]; then
      echo "Failed to ssh into the jump host"
      exit 1
    fi
@@ -307,16 +312,16 @@ if [ "${HYPERCONVERGED_DEV:-false}" = "true" ]; then
     echo "HYPERCONVERGED_DEV is true, but we've failed to determine the base genestack directory"
     exit 1
   fi
-  ssh -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -t ${SSH_USERNAME}@${JUMP_HOST_VIP} \
+  ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} \
     "timeout 1m bash -c 'while ! sudo apt update; do sleep 2; done' && sudo apt install -y rsync git"
   echo "Copying the development source code to the jump host"
   rsync -az \
-        -e "ssh -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null" \
+        -e "ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no" \
         --rsync-path="sudo rsync" \
         $(readlink -fn ${SCRIPT_DIR}/../) ${SSH_USERNAME}@${JUMP_HOST_VIP}:/opt/
 fi
 
-ssh -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -t ${SSH_USERNAME}@${JUMP_HOST_VIP} <<EOC
+ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} <<EOC
 set -e
 if ! command -v git &> /dev/null; then
   echo "git could not be found, installing..."
@@ -472,6 +477,8 @@ if [ ! -f "/etc/genestack/helm-configs/glance/glance-helm-overrides.yaml" ]; the
 cat > /etc/genestack/helm-configs/glance/glance-helm-overrides.yaml <<EOF
 conf:
   glance:
+    DEFAULT:
+      workers: 2
     oslo_messaging_notifications:
       driver: noop
   glance_api_uwsgi:
@@ -620,7 +627,7 @@ fi
 EOC
 
 # Run host and K8S setup
-ssh -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -t ${SSH_USERNAME}@${JUMP_HOST_VIP} <<EOC
+ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} <<EOC
 set -e
 if [ ! -f "/usr/local/bin/queue_max.sh" ]; then
   python3 -m venv ~/.venvs/genestack
@@ -652,7 +659,7 @@ popd
 EOC
 
 # Run Genestack Infrastucture/OpenStack Setup
-ssh -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -t ${SSH_USERNAME}@${JUMP_HOST_VIP} <<EOC
+ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} <<EOC
 set -e
 echo "Installing OpenStack Infrastructure"
 sudo LONGHORN_STORAGE_REPLICAS=1 \
@@ -665,7 +672,7 @@ sudo /opt/genestack/bin/setup-openstack.sh
 EOC
 
 # Run Genestack post setup
-ssh -o StrictHostKeyChecking=no -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -t ${SSH_USERNAME}@${JUMP_HOST_VIP} <<EOC
+ssh -o ForwardAgent=yes -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -t ${SSH_USERNAME}@${JUMP_HOST_VIP} <<EOC
 set -e
 sudo bash <<HERE
 sudo /opt/genestack/bin/setup-openstack-rc.sh