Add GH workflow for running integration tests against VMs (curl installer + update)

Signed-off-by: Tobias K <6317548+theCalcaholic@users.noreply.github.com>

6 files changed, 414 insertions, 14 deletions

diff --git a/.github/actions/create-test-instance/action.yml b/.github/actions/create-test-instance/action.yml
new file mode 100644
index 00000000..0894a397
--- /dev/null
+++ b/.github/actions/create-test-instance/action.yml
@@ -0,0 +1,35 @@
+name: Create Test VM
+description: Create NCP instance for testing in the Hetzner cloud
+inputs:
+  version:
+    description: version (git rev / tag / branch) to install
+    required: true
+  uid:
+    description: A unique ID for labeling/naming generated resources
+    required: true
+  hcloud_token:
+    description: A auth token for Hetzner cloud
+    required: true
+  server_type:
+    description: Server type to use for hetzner servers
+    required: true
+    default: "cx11"
+
+outputs:
+  server_address:
+    description: Adress of the test instance
+  snapshot_id:
+    description: ID of the generated postinstall snapshot
+  test_server_id:
+    description: ID of the created test server
+runs:
+  using: docker
+  image: docker://thecalcaholic/ncp-test-automation
+
+  env:
+    HCLOUD_TOKEN: ${{ inputs.hcloud_token }}
+    UID: ${{ inputs.uid }}
+    SERVER_TYPE: ${{ inputs.server_type }}
+  args:
+    - /ncp-test-automation/bin/actions/create-test-instance.sh
+    - ${{ inputs.version }}
\ No newline at end of file
diff --git a/.github/workflows/vm-tests.yml b/.github/workflows/vm-tests.yml
new file mode 100644
index 00000000..335c220f
--- /dev/null
+++ b/.github/workflows/vm-tests.yml
@@ -0,0 +1,350 @@
+name: 'VM Integration Tests'
+
+on:
+  workflow_dispatch:
+    inputs:
+      version:
+        description: git ref, branch or tag to test against
+        required: false
+        type: string
+  push:
+    branches:
+      - master
+      - devel
+    tags:
+      - v*
+  pull_request:
+
+jobs:
+  setup-installation-test-instance:
+    runs-on: ubuntu-latest
+    outputs:
+      server_address: ${{ steps.create-test-instance.outputs.server_address }}
+      snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
+      test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
+      version: ${{ env.VERSION }}
+    env:
+      VERSION: "${{ github.event.inputs.version || github.head_ref || github.ref_name }}"
+    steps:
+      - uses: actions/checkout@v3
+      - run: |
+          set -e
+          mkdir -p ./.ssh
+          ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
+      - name: upload ssh private key to artifact store
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.run_id }}-install-ssh-privkey
+          path: .ssh
+          if-no-files-found: error
+      - id: create-test-instance
+        uses: ./.github/actions/create-test-instance
+        with:
+          version: ${{ env.VERSION }}
+          uid: "${{ github.run_id }}-install"
+          hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
+          server_type: "cx11"
+
+  setup-update-test-instance:
+    runs-on: ubuntu-latest
+    outputs:
+      server_address: ${{ steps.create-test-instance.outputs.server_address }}
+      snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
+      test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
+      previous_version: ${{ steps.find-version.outputs.previous_version }}
+      version: ${{ env.VERSION }}
+    env:
+      VERSION: "${{ github.event.inputs.version || github.head_ref || github.ref_name }}"
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - shell: bash
+        id: find-version
+        run: |
+          set -e
+          if [[ -n "${{ github.base_ref }}" ]]
+          then
+            version="${{ github.base_ref }}"
+          elif [[ "${{ github.ref }}" == "refs/heads/devel" ]]
+          then
+            version="master"
+          else
+            git fetch -fu --tags origin ${{ github.ref }}:${{ github.ref }}
+            version="$(git describe --tags)"
+            [[ "$version" =~ .*-.*-.* ]] || {
+              git checkout HEAD~1
+              version="$(git describe --tags)"
+            }
+            version="${version%-*-*}"
+          fi
+          echo "Previous version is '$version'"
+          echo "::set-output name=previous_version::${version}"
+      - run: |
+          set -x
+          mkdir -p ./.ssh
+          ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
+      - name: upload ssh private key to artifact store
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ github.run_id }}-update-ssh-privkey
+          path: .ssh
+          if-no-files-found: error
+      - id: create-test-instance
+        uses: ./.github/actions/create-test-instance
+        with:
+          version: "${{ steps.find-version.outputs.previous_version }}"
+          uid: "${{ github.run_id }}-update"
+          hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
+          server_type: "cx11"
+
+  run-installation-test:
+    needs:
+      - setup-installation-test-instance
+    runs-on: ubuntu-latest
+
+    container:
+      image: thecalcaholic/ncp-test-automation:latest
+      env:
+        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
+        UID: "${{ github.run_id }}-install"
+    env:
+      VERSION: ${{ needs.setup-installation-test-instance.outputs.version }}
+      SERVER_ADDRESS: "${{ needs.setup-installation-test-instance.outputs.server_address }}"
+      SNAPSHOT_ID: "${{ needs.setup-installation-test-instance.outputs.snapshot_id }}"
+      HOME: /root
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: 'theCalcaholic/ncp-test-automation'
+      - name: download ssh private key from artifact store
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ github.run_id }}-install-ssh-privkey
+          path: .ssh
+      - name: Test postinstall VM
+        run: |
+          set -e
+          echo "Setup ssh"
+          chmod 0600 ./.ssh/automation_ssh_key
+          eval "$(ssh-agent)"
+          ssh-add ./.ssh/automation_ssh_key
+          
+          cd bin
+          source ./library.sh
+          
+          trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
+          
+          setup-ssh-port-forwarding "$SERVER_ADDRESS"
+          
+          echo "Run integration tests"
+          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
+    
+            echo "Integration tests failed"
+            echo "Here are the last lines of ncp-install.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
+            echo "==========================================="
+            echo "and ncp.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
+            echo "==========================================="
+            exit 1
+          }
+
+  run-update-test:
+    needs:
+      - setup-update-test-instance
+    runs-on: ubuntu-latest
+
+    container:
+      image: thecalcaholic/ncp-test-automation:latest
+      env:
+        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
+        UID: "${{ github.run_id }}-update"
+    env:
+      PREVIOUS_VERSION: ${{ needs.setup-update-test-instance.outputs.previous_version }}
+      VERSION: ${{ needs.setup-update-test-instance.outputs.version }}
+      SERVER_ADDRESS: "${{ needs.setup-update-test-instance.outputs.server_address }}"
+      SNAPSHOT_ID: "${{ needs.setup-update-test-instance.outputs.snapshot_id }}"
+      HOME: /root
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          repository: 'theCalcaholic/ncp-test-automation'
+      - name: download ssh private key from artifact store
+        uses: actions/download-artifact@v3
+        with:
+          name: ${{ github.run_id }}-update-ssh-privkey
+          path: .ssh
+      - name: perform update
+        run: |
+          set -e
+          
+          echo "Setup ssh"
+          chmod 0600 ./.ssh/automation_ssh_key
+          eval "$(ssh-agent)"
+          ssh-add ./.ssh/automation_ssh_key
+          
+          . ./bin/library.sh
+
+          echo "Updating from $PREVIOUS_VERSION to $VERSION"
+          ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true
+          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'"
+      - name: Run integration tests
+        run: |
+          set -e
+          
+          echo "Setup ssh"
+          eval "$(ssh-agent)"
+          ssh-add ./.ssh/automation_ssh_key
+          
+          cd bin
+          source ./library.sh
+          
+          trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
+          
+          echo "Run integration tests"
+          setup-ssh-port-forwarding "$SERVER_ADDRESS"
+          
+          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
+          
+            echo "Integration tests failed"
+            echo "Here are the last lines of ncp-install.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
+            echo "==========================================="
+            echo "and ncp.log:"
+            echo "==========================================="
+            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
+            echo "==========================================="
+            exit 1
+          }
+
+  create-postactivation-snapshots:
+    if: ${{ always() }}
+    needs:
+      - setup-installation-test-instance
+      - setup-update-test-instance
+      - run-installation-test
+      - run-update-test
+    runs-on: ubuntu-latest
+    container:
+      image: thecalcaholic/ncp-test-automation:latest
+      env:
+        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
+    strategy:
+      matrix:
+        test_type: [install, update]
+        include:
+          - test_type: install
+            server_address: ${{ needs.setup-installation-test-instance.outputs.server_address }}
+            test_result: ${{ needs.setup-installation-test-instance.result }}
+            test_server_id: ${{ needs.setup-installation-test-instance.outputs.test_server_id }}
+            version: ${{ needs.setup-installation-test-instance.outputs.version }}
+          - test_type: update
+            server_address: ${{ needs.setup-update-test-instance.outputs.server_address }}
+            test_result: ${{ needs.setup-update-test-instance.result }}
+            test_server_id: ${{ needs.setup-update-test-instance.outputs.test_server_id }}
+            version: ${{ needs.setup-update-test-instance.outputs.version }}
+      fail-fast: false
+
+    env:
+      UID: ${{ github.run_id }}-${{ matrix.test_type }}
+      VERSION: ${{ matrix.version }}
+    steps:
+      - name: download ssh private key from artifact store
+        uses: actions/download-artifact@v3
+        if: ${{ contains('success|failure', matrix.test_result) }}
+        with:
+          name: ${{ github.run_id }}-${{ matrix.test_type }}-ssh-privkey
+          path: /github/workspace/.ssh
+      - name: Shutdown server
+        if: ${{ contains('success|failure', matrix.test_result) }}
+        run: |
+          chmod 0600 /github/workspace/.ssh/automation_ssh_key
+          export SSH_PUBLIC_KEY="$(cat /github/workspace/.ssh/automation_ssh_key.pub)"
+          bash /ncp-test-automation/bin/entrypoint.sh
+          eval "$(ssh-agent)"
+          ssh-add /github/workspace/.ssh/automation_ssh_key
+  
+          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${{ matrix.server_address }}" <<EOF
+          systemctl stop mariadb
+          systemctl poweroff
+          EOF
+      - name: Create Snapshot
+        if: ${{ contains('success|failure', matrix.test_result) }}
+        shell: bash
+        run: |
+          set -x
+          echo "${{ needs.setup-installation-test-instance.outputs.test_server_id }}"
+          echo "${{ needs.setup-update-test-instance.outputs.test_server_id }}"
+          echo "${{ matrix.test_server_id }}"
+          cd /ncp-test-automation/bin
+          
+          . ./library.sh
+          
+          tf-init "$TF_SNAPSHOT"
+          tf-apply "$TF_SNAPSHOT" "$TF_VAR_FILE" -var="branch=${{ matrix.version }}" -var="snapshot_provider_id=${{ matrix.test_server_id }}" -var="snapshot_type=ncp-postactivation" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate"
+          snapshot_id="$(tf-output "$TF_SNAPSHOT" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate" snapshot_id)"
+          hcloud image add-label -o "$snapshot_id" "test-result=${{ matrix.test_result }}"
+
+  cleanup:
+    if: ${{ always() }}
+    needs:
+      - create-postactivation-snapshots
+    runs-on: ubuntu-latest
+    container:
+      image: thecalcaholic/ncp-test-automation:latest
+      env:
+        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
+    strategy:
+      matrix:
+        uid: ["${{ github.run_id }}-install", "${{ github.run_id }}-update"]
+      fail-fast: false
+    env:
+      HOME: '/root'
+      UID: ${{ matrix.uid }}
+    defaults:
+      run:
+        shell: bash
+        working-directory: /ncp-test-automation/bin
+    steps:
+      - name: Teardown VMs
+        run: |
+          for server in $(hcloud server list -o noheader -o columns=id -l "ci=${UID}")
+          do
+            echo "Deleting server '$server'..."
+            hcloud server delete "$server"
+            echo "done."
+          done
+      - name: Delete ssh key
+        run: |
+          source ./library.sh
+          hcloud-clear-root-key
+  cleanup-snapshots:
+    if: ${{ always() }}
+    needs:
+      - cleanup
+    runs-on: ubuntu-latest
+    container:
+      image: thecalcaholic/ncp-test-automation:latest
+      env:
+        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
+    env:
+      HOME: '/root'
+    steps:
+      - name: Delete old snapshots
+        run: |
+          for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -20)
+          do
+            echo "Deleting snapshot '$snapshot'..."
+            hcloud image delete "$snapshot"
+            echo "done."
+          done
diff --git a/README.md b/README.md
index bde58d21..71d82eeb 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,8 @@ This code also generates the NextCloudPi [docker image](https://hub.docker.com/r
 
 Find the full documentation at [docs.nextcloudpi.com](http://docs.nextcloudpi.com)
 
+[![VM Integration Tests](https://github.com/nextcloud/nextcloudpi/actions/workflows/vm-tests.yml/badge.svg)](https://github.com/nextcloud/nextcloudpi/actions/workflows/vm-tests.yml)
+
 ## Features
 
  * Debian/Raspbian 11 Bullseye
diff --git a/tests/activation_tests.py b/tests/activation_tests.py
index 99311af2..bbc59a66 100755
--- a/tests/activation_tests.py
+++ b/tests/activation_tests.py
@@ -24,6 +24,7 @@ from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.firefox.options import Options
 from selenium.common.exceptions import UnexpectedAlertPresentException
 from selenium.common.exceptions import NoSuchElementException
 from selenium.common.exceptions import TimeoutException
@@ -88,12 +89,12 @@ def signal_handler(sig, frame):
         sys.exit(0)
 
 
-def test_activation(IP, nc_port, admin_port):
+def test_activation(IP, nc_port, admin_port, options):
     """ Activation process checks"""
 
     # activation page
     test = Test()
-    driver = webdriver.Firefox(service_log_path='/dev/null')
+    driver = webdriver.Firefox(options=options)
     driver.implicitly_wait(5)
     test.new("activation opens")
     driver.get(f"https://{IP}:{nc_port}")
@@ -134,7 +135,7 @@ def test_activation(IP, nc_port, admin_port):
 
     # ncp-web
     test.new("ncp-web")
-    driver = webdriver.Firefox(service_log_path='/dev/null')
+    driver = webdriver.Firefox(options=options)
     try:
         driver.get(f"https://ncp:{urllib.parse.quote_plus(ncp_pass)}@{IP}:{admin_port}")
     except UnexpectedAlertPresentException:
@@ -150,15 +151,18 @@ if __name__ == "__main__":
 
     # parse options
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help'])
+        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help', 'no-gui'])
     except getopt.GetoptError:
         usage()
         sys.exit(2)
 
+    options = Options()
     for opt, arg in opts:
         if opt in ('-h', '--help'):
             usage()
             sys.exit(2)
+        elif opt == '--no-gui':
+            options.headless = True
         else:
             usage()
             sys.exit(2)
@@ -170,7 +174,8 @@ if __name__ == "__main__":
     admin_port = args[2] if len(args) > 2 else "4443"
     print("Activation tests " + tc.yellow + IP + tc.normal)
     print("---------------------------")
-    test_activation(IP, nc_port, admin_port)
+
+    test_activation(IP, nc_port, admin_port, options)
 
 # License
 #
diff --git a/tests/nextcloud_tests.py b/tests/nextcloud_tests.py
index d176fe76..d4b4dea8 100755
--- a/tests/nextcloud_tests.py
+++ b/tests/nextcloud_tests.py
@@ -25,6 +25,7 @@ from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.firefox.options import Options
 from selenium.common.exceptions import NoSuchElementException, WebDriverException, TimeoutException
 from typing import List, Tuple
 import traceback
@@ -175,11 +176,12 @@ if __name__ == "__main__":
 
     # parse options
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help'])
+        opts, args = getopt.getopt(sys.argv[1:], 'hn', ['help', 'new', 'no-gui'])
     except getopt.GetoptError:
         usage()
         sys.exit(2)
 
+    options = Options()
     for opt, arg in opts:
         if opt in ('-h', '--help'):
             usage()
@@ -187,6 +189,8 @@ if __name__ == "__main__":
         elif opt in ('-n', '--new'):
             if os.path.exists(test_cfg):
                 os.unlink(test_cfg)
+        elif opt == '--no-gui':
+            options.headless = True
         else:
             usage()
             sys.exit(2)
@@ -222,7 +226,7 @@ if __name__ == "__main__":
     print("Nextcloud tests " + tc.yellow + IP + tc.normal)
     print("---------------------------")
 
-    driver = webdriver.Firefox(service_log_path='/dev/null')
+    driver = webdriver.Firefox(service_log_path='/dev/null', options=options)
     try:
         test_nextcloud(IP, nc_port, driver)
     finally:
diff --git a/tests/system_tests.py b/tests/system_tests.py
index a7eea694..e722b551 100755
--- a/tests/system_tests.py
+++ b/tests/system_tests.py
@@ -181,15 +181,18 @@ if __name__ == "__main__":
 
     # parse options
     try:
-        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help'])
+        opts, args = getopt.getopt(sys.argv[1:], 'h', ['help', 'no-ping'])
     except getopt.GetoptError:
         usage()
         sys.exit(2)
 
+    skip_ping = False
     for opt, arg in opts:
         if opt in ('-h', '--help'):
             usage()
             sys.exit(2)
+        elif opt == '--no-ping':
+            skip_ping = True
         else:
             usage()
             sys.exit(2)
@@ -241,12 +244,13 @@ if __name__ == "__main__":
         pre_cmd = ['ssh', '-o UserKnownHostsFile=/dev/null' , '-o PasswordAuthentication=no',
                 '-o StrictHostKeyChecking=no', '-o ConnectTimeout=1', ssh_cmd[4:]]
 
-        at_char = ssh_cmd.index('@')
-        ip = ssh_cmd[at_char+1:]
-        ping_cmd = run(['ping', '-c1', '-w1', ip], stdout=PIPE, stderr=PIPE)
-        if ping_cmd.returncode != 0:
-            print(tc.red + "No connectivity to " + tc.yellow + ip + tc.normal)
-            sys.exit(1)
+        if not skip_ping:
+            at_char = ssh_cmd.index('@')
+            ip = ssh_cmd[at_char+1:]
+            ping_cmd = run(['ping', '-c1', '-w10', ip], stdout=PIPE, stderr=PIPE)
+            if ping_cmd.returncode != 0:
+                print(tc.red + "No connectivity to " + tc.yellow + ip + tc.normal)
+                #sys.exit(1)
 
         ssh_test = run(pre_cmd + [':'], stdout=PIPE, stderr=PIPE)
         if ssh_test.returncode != 0: