path: root/.github/workflows/vm-tests.yml

name: 'VM Integration Tests'

on:
  workflow_dispatch:
    inputs:
      git_ref:
        description: git ref, branch or tag to test against
        required: false
        type: string
  workflow_call:
    inputs:
      git_ref:
        description: git ref, branch or tag to test against
        required: false
        type: string
  push:
    branches:
      - master

jobs:
  setup-installation-test-instance:
    runs-on: ubuntu-latest
    outputs:
      server_address: ${{ steps.create-test-instance.outputs.server_address }}
      snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
      test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
      version: ${{ env.VERSION }}
    env:
      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
    steps:
      - uses: actions/checkout@v3
      - run: |
          set -e
          mkdir -p ./.ssh
          ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
      - name: upload ssh private key to artifact store
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.run_id }}-install-ssh-privkey
          path: .ssh
          if-no-files-found: error
      - id: create-test-instance
        uses: ./.github/actions/create-test-instance
        with:
          version: ${{ env.VERSION }}
          uid: "${{ github.run_id }}-install"
          hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
          server_type: "cx11"

  setup-update-test-instance:
    runs-on: ubuntu-latest
    outputs:
      server_address: ${{ steps.create-test-instance.outputs.server_address }}
      snapshot_id: ${{ steps.create-test-instance.outputs.snapshot_id }}
      test_server_id: ${{ steps.create-test-instance.outputs.test_server_id }}
      previous_version: ${{ steps.find-version.outputs.previous_version }}
      version: ${{ env.VERSION }}
    env:
      VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}"
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - shell: bash
        id: find-version
        run: |
          set -e
          if [[ -n "${{ github.base_ref }}" ]]
          then
            version="${{ github.base_ref }}"
          elif [[ "${{ github.ref }}" == "refs/heads/devel" ]]
          then
            version="master"
          else
            git fetch -fu --tags origin ${{ github.ref }}:${{ github.ref }}
            version="$(git describe --tags)"
            [[ "$version" =~ .*-.*-.* ]] || {
              git checkout HEAD~1
              version="$(git describe --tags)"
            }
            version="${version%-*-*}"
          fi
          echo "Previous version is '$version'"
          echo "{previous_version}={${version}}" >> $GITHUB_OUTPUT
      - run: |
          set -x
          mkdir -p ./.ssh
          ssh-keygen -t ed25519 -f ".ssh/automation_ssh_key"
      - name: upload ssh private key to artifact store
        uses: actions/upload-artifact@v3
        with:
          name: ${{ github.run_id }}-update-ssh-privkey
          path: .ssh
          if-no-files-found: error
      - id: create-test-instance
        uses: ./.github/actions/create-test-instance
        with:
          version: "${{ steps.find-version.outputs.previous_version }}"
          uid: "${{ github.run_id }}-update"
          hcloud_token: ${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}
          server_type: "cx11"

  run-installation-test:
    needs:
      - setup-installation-test-instance
    runs-on: ubuntu-latest

    container:
      image: thecalcaholic/ncp-test-automation:latest
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
        UID: "${{ github.run_id }}-install"
    env:
      VERSION: ${{ needs.setup-installation-test-instance.outputs.version }}
      SERVER_ADDRESS: "${{ needs.setup-installation-test-instance.outputs.server_address }}"
      SNAPSHOT_ID: "${{ needs.setup-installation-test-instance.outputs.snapshot_id }}"
      HOME: /root
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v3
        with:
          repository: 'theCalcaholic/ncp-test-automation'
      - name: download ssh private key from artifact store
        uses: actions/download-artifact@v3
        with:
          name: ${{ github.run_id }}-install-ssh-privkey
          path: .ssh
      - name: Test postinstall VM
        run: |
          set -e
          echo "Setup ssh"
          chmod 0600 ./.ssh/automation_ssh_key
          eval "$(ssh-agent)"
          ssh-add ./.ssh/automation_ssh_key
          
          cd bin
          source ./library.sh
          
          trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
          
          setup-ssh-port-forwarding "$SERVER_ADDRESS"
          
          echo "Run integration tests"
          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
    
            echo "Integration tests failed"
            echo "Here are the last lines of ncp-install.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
            echo "==========================================="
            echo "and ncp.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
            echo "==========================================="
            exit 1
          }

  run-update-test:
    needs:
      - setup-update-test-instance
    runs-on: ubuntu-latest

    container:
      image: thecalcaholic/ncp-test-automation:latest
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
        UID: "${{ github.run_id }}-update"
    env:
      PREVIOUS_VERSION: ${{ needs.setup-update-test-instance.outputs.previous_version }}
      VERSION: ${{ needs.setup-update-test-instance.outputs.version }}
      SERVER_ADDRESS: "${{ needs.setup-update-test-instance.outputs.server_address }}"
      SNAPSHOT_ID: "${{ needs.setup-update-test-instance.outputs.snapshot_id }}"
      HOME: /root
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v3
        with:
          repository: 'theCalcaholic/ncp-test-automation'
      - name: download ssh private key from artifact store
        uses: actions/download-artifact@v3
        with:
          name: ${{ github.run_id }}-update-ssh-privkey
          path: .ssh
      - name: perform update
        run: |
          set -e
          
          echo "Setup ssh"
          chmod 0600 ./.ssh/automation_ssh_key
          eval "$(ssh-agent)"
          ssh-add ./.ssh/automation_ssh_key
          
          . ./bin/library.sh

          echo "Updating from $PREVIOUS_VERSION to $VERSION"
          ssh-keygen -f "$HOME/.ssh/known_hosts" -R "${SERVER_ADDRESS}" 2> /dev/null || true
          ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" "ncp-update '$VERSION'"
      - name: Run integration tests
        run: |
          set -e
          
          echo "Setup ssh"
          eval "$(ssh-agent)"
          ssh-add ./.ssh/automation_ssh_key
          
          cd bin
          source ./library.sh
          
          trap 'terminate-ssh-port-forwarding "${SERVER_ADDRESS}"' EXIT 1 2
          
          echo "Run integration tests"
          setup-ssh-port-forwarding "$SERVER_ADDRESS"
          
          test-ncp-instance -a -f "$SNAPSHOT_ID" -b "${VERSION}" "root@${SERVER_ADDRESS}" "localhost" "8443" "9443" || {
          
            echo "Integration tests failed"
            echo "Here are the last lines of ncp-install.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp-install.log;
            echo "==========================================="
            echo "and ncp.log:"
            echo "==========================================="
            ssh "${SSH_OPTIONS[@]}" "root@${SERVER_ADDRESS}" tail /var/log/ncp.log;
            echo "==========================================="
            exit 1
          }

  install-postactivation-snapshot:
    if: ${{ always() }}
    needs:
      - setup-installation-test-instance
      - run-installation-test
    runs-on: ubuntu-latest
    container:
      image: thecalcaholic/ncp-test-automation:latest
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"

    env:
      TEST_TYPE: install
      SERVER_ADDRESS: ${{ needs.setup-installation-test-instance.outputs.server_address }}
      TEST_RESULT: ${{ needs.setup-installation-test-instance.result }}
      TEST_SERVER_ID: ${{ needs.setup-installation-test-instance.outputs.test_server_id }}
      VERSION: ${{ needs.setup-installation-test-instance.outputs.version }}

      UID: ${{ github.run_id }}-install
    steps:
      - name: download ssh private key from artifact store
        uses: actions/download-artifact@v3
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        with:
          name: ${{ github.run_id }}-${{ env.TEST_TYPE }}-ssh-privkey
          path: /github/workspace/.ssh
      - name: Shutdown server
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        run: |
          chmod 0600 /github/workspace/.ssh/automation_ssh_key
          export SSH_PUBLIC_KEY="$(cat /github/workspace/.ssh/automation_ssh_key.pub)"
          bash /ncp-test-automation/bin/entrypoint.sh
          eval "$(ssh-agent)"
          ssh-add /github/workspace/.ssh/automation_ssh_key
  
          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS?}" <<EOF
          systemctl stop mariadb
          systemctl poweroff
          EOF
      - name: Create Snapshot
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        shell: bash
        run: |
          set -x
          echo "${{ needs.setup-installation-test-instance.outputs.test_server_id }}"
          echo "${TEST_SERVER_ID?}"
          cd /ncp-test-automation/bin
          
          . ./library.sh
          
          tf-init "$TF_SNAPSHOT"
          tf-apply "$TF_SNAPSHOT" "$TF_VAR_FILE" -var="branch=${VERSION?}" -var="snapshot_provider_id=${TEST_SERVER_ID?}" -var="snapshot_type=ncp-postactivation" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate"
          snapshot_id="$(tf-output "$TF_SNAPSHOT" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate" snapshot_id)"
          hcloud image add-label -o "$snapshot_id" "test-result=${TEST_RESULT?}"

  update-postactivation-snapshot:
    if: ${{ always() }}
    needs:
      - setup-update-test-instance
      - run-update-test
    runs-on: ubuntu-latest
    container:
      image: thecalcaholic/ncp-test-automation:latest
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"

    env:
      TEST_TYPE: update
      SERVER_ADDRESS: ${{ needs.setup-update-test-instance.outputs.server_address }}
      TEST_RESULT: ${{ needs.setup-update-test-instance.result }}
      TEST_SERVER_ID: ${{ needs.setup-update-test-instance.outputs.test_server_id }}
      VERSION: ${{ needs.setup-update-test-instance.outputs.version }}
      UID: ${{ github.run_id }}-update
    steps:
      - name: download ssh private key from artifact store
        uses: actions/download-artifact@v3
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        with:
          name: ${{ github.run_id }}-${{ env.TEST_TYPE }}-ssh-privkey
          path: /github/workspace/.ssh
      - name: Shutdown server
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        run: |
          chmod 0600 /github/workspace/.ssh/automation_ssh_key
          export SSH_PUBLIC_KEY="$(cat /github/workspace/.ssh/automation_ssh_key.pub)"
          bash /ncp-test-automation/bin/entrypoint.sh
          eval "$(ssh-agent)"
          ssh-add /github/workspace/.ssh/automation_ssh_key
          
          ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${SERVER_ADDRESS?}" <<EOF
          systemctl stop mariadb
          systemctl poweroff
          EOF
      - name: Create Snapshot
        if: ${{ contains('success|failure', env.TEST_RESULT) }}
        shell: bash
        run: |
          set -x
          echo "${{ needs.setup-update-test-instance.outputs.test_server_id }}"
          echo "${TEST_SERVER_ID?}"
          cd /ncp-test-automation/bin
          
          . ./library.sh
          
          tf-init "$TF_SNAPSHOT"
          tf-apply "$TF_SNAPSHOT" "$TF_VAR_FILE" -var="branch=${VERSION?}" -var="snapshot_provider_id=${TEST_SERVER_ID?}" -var="snapshot_type=ncp-postactivation" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate"
          snapshot_id="$(tf-output "$TF_SNAPSHOT" -state="${TF_SNAPSHOT}/${VERSION//\//.}.postactivation.tfstate" snapshot_id)"
          hcloud image add-label -o "$snapshot_id" "test-result=${TEST_RESULT?}"

  cleanup:
    if: ${{ always() }}
    needs:
      - install-postactivation-snapshot
      - update-postactivation-snapshot
    runs-on: ubuntu-latest
    container:
      image: thecalcaholic/ncp-test-automation:latest
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
    strategy:
      matrix:
        uid: ["${{ github.run_id }}-install", "${{ github.run_id }}-update"]
      fail-fast: false
    env:
      HOME: '/root'
      UID: ${{ matrix.uid }}
    defaults:
      run:
        shell: bash
        working-directory: /ncp-test-automation/bin
    steps:
      - name: Teardown VMs
        run: |
          for server in $(hcloud server list -o noheader -o columns=id -l "ci=${UID?}")
          do
            echo "Deleting server '$server'..."
            hcloud server delete "$server"
            echo "done."
          done
      - name: Delete ssh key
        run: |
          source ./library.sh
          hcloud-clear-root-key
  cleanup-snapshots:
    if: ${{ always() }}
    needs:
      - install-postactivation-snapshot
      - update-postactivation-snapshot
    runs-on: ubuntu-latest
    container:
      image: thecalcaholic/ncp-test-automation:latest
      env:
        HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}"
    env:
      HOME: '/root'
    steps:
      - name: Delete old snapshots
        run: |
          for snapshot in $(hcloud image list -t snapshot -o noheader -o columns=id | head -n -20)
          do
            echo "Deleting snapshot '$snapshot'..."
            hcloud image delete "$snapshot"
            echo "done."
          done