From 7af7582fffe521d166fc6bcf6999602a29131337 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tobias=20Kn=C3=B6ppler?= <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 20 Jul 2022 00:25:19 +0200 Subject: Create releases from GH workflow (#1507) - Implement lxd-build GH workflow - Implement GH workflow for building armbian and raspberry pi images - Implement release GH workflow --- .github/workflows/build-docker.yml | 57 ++++----------- .github/workflows/build-lxd.yml | 105 ++++++++++++++++++++++++++++ .github/workflows/build-sd-images.yml | 97 ++++++++++++++++++++++++++ .github/workflows/release.yml | 126 ++++++++++++++++++++++++++++++++++ .github/workflows/vm-tests.yml | 15 ++-- bin/ncp/NETWORKING/dnsmasq.sh | 3 +- build/build-LXD.sh | 14 ++-- build/build-SD-armbian.sh | 13 ++-- build/build-SD-rpi.sh | 6 +- build/buildlib.sh | 9 ++- tests/nextcloud_tests.py | 37 ++++++---- 11 files changed, 404 insertions(+), 78 deletions(-) create mode 100644 .github/workflows/build-lxd.yml create mode 100644 .github/workflows/build-sd-images.yml create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 9799c237..c8e2c202 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -5,11 +5,15 @@ name: 'Docker Integration Tests and Release' on: + workflow_call: + inputs: + git_ref: + required: true + type: string push: branches: - "**" - tags: - - 'v*' + pull_request: jobs: build: @@ -21,6 +25,8 @@ jobs: - armhf - arm64 fail-fast: false + env: + VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" steps: - name: Setup Docker Buildx uses: docker/setup-buildx-action@v1 @@ -30,6 +36,8 @@ jobs: - name: Checkout code uses: actions/checkout@v3 + with: + ref: "${{ env.VERSION }}" - name: Login to docker run: | @@ -50,6 +58,8 @@ jobs: needs: - build runs-on: ubuntu-latest + env: + VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" strategy: matrix: @@ -71,6 +81,8 @@ jobs: - name: Checkout code uses: actions/checkout@v3 + with: + ref: "${{ env.VERSION }}" - name: Setup Firefox uses: browser-actions/setup-firefox@latest @@ -116,44 +128,3 @@ jobs: } echo "Nextcloud test successful" - release: - needs: - - test - if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} - runs-on: ubuntu-latest - steps: - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Checkout code - uses: actions/checkout@v3 - - - name: Create manifest and push as tag to docker hub - run: | - . ./build/buildlib.sh - - for arch in x86 armhf arm64 - do - docker pull "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}" - docker tag "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}" "ownyourbits/nextcloudpi-${arch}:${version?}" - docker tag "ownyourbits/nextcloudpi-${arch}:${version?}" "ownyourbits/nextcloudpi-${arch}:latest" - docker push "ownyourbits/nextcloudpi-${arch}:${version?}" - docker push "ownyourbits/nextcloudpi-${arch}:latest" - done - - docker manifest create ownyourbits/nextcloudpi:${version?} \ - ownyourbits/nextcloudpi-armhf:${version?} \ - ownyourbits/nextcloudpi-x86:${version?} \ - ownyourbits/nextcloudpi-arm64:${version?} - docker manifest push ownyourbits/nextcloudpi:${version?} - - - name: Create manifest and push as latest to docker hub - run: | - docker manifest create ownyourbits/nextcloudpi:latest \ - ownyourbits/nextcloudpi-armhf:latest \ - ownyourbits/nextcloudpi-x86:latest \ - ownyourbits/nextcloudpi-arm64:latest - docker manifest push ownyourbits/nextcloudpi:latest diff --git a/.github/workflows/build-lxd.yml b/.github/workflows/build-lxd.yml new file mode 100644 index 00000000..3423ce70 --- /dev/null +++ b/.github/workflows/build-lxd.yml @@ -0,0 +1,105 @@ +name: "Build and test LXD image" +on: + workflow_call: + inputs: + git_ref: + required: true + type: string + outputs: + artifact_name: + value: "${{ jobs.build-lxd.outputs.artifact_name }}" + push: + branches: + - "**" + pull_request: + +jobs: + build-lxd: + runs-on: ubuntu-latest + outputs: + artifact_name: ${{ steps.pack-lxd.outputs.artifact_name }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + - uses: whywaita/setup-lxd@v1 + with: + lxd_version: latest/stable + - name: Build LXD image + id: build-lxd + run: | + ./build/build-LXD.sh + - name: Pack LXD image + id: pack-lxd + run: | + . ./build/buildlib.sh + ARTIFACT_NAME="NextCloudPi_LXD_$( date "+%m-%d-%y" )" + lxc image export -q ncp/"${version}" "output/${ARTIFACT_NAME}" + echo "::set-output name=artifact_name::${ARTIFACT_NAME}.tar.gz" + - name: upload LXD image to artifact store + uses: actions/upload-artifact@v3 + with: + name: ${{ github.run_id }}-lxd-image + path: output/${{ steps.pack-lxd.outputs.artifact_name }} + if-no-files-found: error + + test-lxd: + needs: + - build-lxd + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + - uses: whywaita/setup-lxd@v1 + with: + lxd_version: latest/stable + - name: Setup Firefox + uses: browser-actions/setup-firefox@latest + - name: Setup GeckoDriver + uses: browser-actions/setup-geckodriver@latest + - name: Setup Selenium + run: pip install selenium + - name: download LXD image from artifact store + uses: actions/download-artifact@v3 + with: + name: ${{ github.run_id }}-lxd-image + - name: Launch ncp container + run: | + set -x + lxc delete -q -f ncp || true + lxc image import -q "./${{ needs.build-lxd.outputs.artifact_name }}" --alias "ncp/test" + systemd-run --user --scope -p "Delegate=yes" lxc launch -q "ncp/test" ncp + lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' + sleep 30 + ip="$(lxc list -c n4 -f csv | grep '^ncp' | cut -d ',' -f2)" + ip="${ip/% *}" + echo "${ip} nextcloudpi.local" | sudo tee /etc/hosts + - name: Test LXD Image + working-directory: ./tests + run: | + python activation_tests.py --no-gui "nextcloudpi.local" 443 4443 || { + echo "Activation test failed!" + echo "Geckodriver logs:" + tail -n 20 geckodriver.log >&2 || true + echo "================" + echo "ncp.log: " + lxc exec ncp -- "tail -n20 /var/log/ncp.log" + exit 1 + } + python system_tests.py --non-interactive || { + echo "System test failed!" + exit 1 + } + python nextcloud_tests.py --no-gui "nextcloudpi.local" 443 4443 || { + echo "Nextcloud test failed!" + echo "Geckodriver logs:" + tail -n 20 geckodriver.log >&2 || true + echo "================" + echo "ncp.log: " + lxc exec ncp -- "tail -n20 /var/log/ncp.log" + exit 1 + } + lxc stop ncp \ No newline at end of file diff --git a/.github/workflows/build-sd-images.yml b/.github/workflows/build-sd-images.yml new file mode 100644 index 00000000..86d92403 --- /dev/null +++ b/.github/workflows/build-sd-images.yml @@ -0,0 +1,97 @@ +name: "Build SD images" +on: + workflow_call: + inputs: + git_ref: + required: true + type: string + +jobs: + build-rpi: + runs-on: ubuntu-latest + env: + HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}" + UID: "${{ github.run_id }}-rpi" + VERSION: "${{ inputs.git_ref }}" + defaults: + run: + shell: bash + outputs: + artifact_name: ${{ steps.pack-rpi.outputs.artifact_name }} + steps: + - uses: 3bit/setup-hcloud@v1 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: "${{ env.VERSION }}" + - name: Build RPI SD Image + run: | + set -e + wget -q https://github.com/multiarch/qemu-user-static/releases/latest/download/qemu-aarch64-static -O ./qemu-aarch64-static + ./build/build-SD-rpi.sh + - name: Pack RPI Image + id: pack-rpi + run: | + IMG="NextCloudPi_RPi_$( date "+%m-%d-%y" ).img" + TAR="$( basename "$IMG" .img ).tar.gz" + . ./build/buildlib.sh + pack_image "tmp/$IMG" "output/$TAR" + echo "::set-output name=artifact_name::${TAR}" + - name: upload RPI image to artifact store + uses: actions/upload-artifact@v3 + with: + name: ${{ github.run_id }}-rpi-image + path: output/${{ steps.pack-rpi.outputs.artifact_name }} + if-no-files-found: error + + build-armbian: + runs-on: ubuntu-latest + env: + VERSION: "${{ inputs.git_ref }}" + defaults: + run: + shell: bash + outputs: + artifact_name: ${{ steps.pack-armbian.outputs.artifact_name }} + strategy: + matrix: + board: + - odroidxu4 OdroidHC2 + - rockpro64 RockPro64 + - rock64 Rock64 + - bananapi Bananapi + - odroidhc4 OdroidHC4 + - odroidc4 OdroidC4 + - odroidc2 OdroidC2 + fail-fast: false + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: "${{ env.VERSION }}" + - name: "Build Armbian" + run: | + set -x + export LIB_TAG=master + board_params=(${{ matrix.board}}) + ./build/build-SD-armbian.sh "${board_params[@]}" + - name: "Pack image" + id: pack-armbian + run: | + board_params=(${{ matrix.board}}) + IMG="NextCloudPi_${board_params[1]}_$( date "+%m-%d-%y" ).img" + TAR="$( basename "$IMG" .img ).tar.gz" + + artifacts=("armbian/output/images/Armbian"*.img) + mv "${artifacts[0]}" "tmp/$IMG" + . ./build/buildlib.sh + pack_image "tmp/$IMG" "output/$TAR" + echo "::set-output name=artifact_name::${TAR}" + - name: upload Armbian image to artifact store + uses: actions/upload-artifact@v3 + with: + name: ${{ github.run_id }}-armbian-image + path: output/${{ steps.build-pack-armbian.outputs.artifact_name }} + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..c116d277 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,126 @@ +name: 'Release' +on: + workflow_dispatch: + inputs: + git_ref: + description: git ref, branch or tag to test against + required: false + type: string + push: + tags: ["v*"] + +jobs: + build-and-test-lxd: + uses: ./.github/workflows/build-lxd.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + secrets: inherit + + build-sd-images: + uses: ./.github/workflows/build-sd-images.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + secrets: inherit + + build-and-test-docker: + uses: ./.github/workflows/build-docker.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + secrets: inherit + + test-curl-installer: + uses: ./.github/workflows/vm-tests.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + secrets: inherit + + docker-release: + needs: + - build-and-test-docker + - test-curl-installer + if: ${{ github.event_name == 'push' && github.ref_type == 'tag' && false }} + runs-on: ubuntu-latest + steps: + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Create manifest and push as tag to docker hub + run: | + . ./build/buildlib.sh + + for arch in x86 armhf arm64 + do + docker pull "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}" + docker tag "thecalcaholic/ncp-internal-${arch}:${{ github.run_id }}" "ownyourbits/nextcloudpi-${arch}:${version?}" + docker tag "ownyourbits/nextcloudpi-${arch}:${version?}" "ownyourbits/nextcloudpi-${arch}:latest" + docker push "ownyourbits/nextcloudpi-${arch}:${version?}" + docker push "ownyourbits/nextcloudpi-${arch}:latest" + done + + docker manifest create ownyourbits/nextcloudpi:${version?} \ + ownyourbits/nextcloudpi-armhf:${version?} \ + ownyourbits/nextcloudpi-x86:${version?} \ + ownyourbits/nextcloudpi-arm64:${version?} + docker manifest push ownyourbits/nextcloudpi:${version?} + + - name: Create manifest and push as latest to docker hub + run: | + docker manifest create ownyourbits/nextcloudpi:latest \ + ownyourbits/nextcloudpi-armhf:latest \ + ownyourbits/nextcloudpi-x86:latest \ + ownyourbits/nextcloudpi-arm64:latest + docker manifest push ownyourbits/nextcloudpi:latest + + github-release: + needs: + - build-and-test-lxd + - build-sd-images + - test-curl-installer + runs-on: ubuntu-latest + env: + UID: "${{ github.run_id }}-rpi" + VERSION: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + defaults: + run: + shell: bash + steps: + - name: download images from artifact store + uses: actions/download-artifact@v3 + with: + path: artifacts + - name: Create Release + run: | + mkdir -p release + cd release + msg=() + assets=() + for asset in ../artifacts/*/*.tar.gz; + do + mv "$asset" ./ + asset_name="$(basename "$asset")" + msg="$msg + \`\`\` + $(md5sum "$asset_name") + \`\`\`" + assets+=(-a "$asset_name") + done + + echo "Assets: ${assets[*]}" + echo "Message: ${msg[*]}" + - name: Publish + if: ${{ github.event_name == 'push' && github.ref_type == 'tag' }} + run: | + hub release create "${assets[@]}" -F - "${{ env.VERSION }}" < /dev/null 2>&1 || rc=$? - ! is_docker && [[ $rc -eq 3 ]] && [[ "$INIT_SYSTEM" != "chroot" ]] && { + ! is_docker && [[ $rc -eq 3 ]] && ! [[ "$INIT_SYSTEM" =~ ^("chroot"|"unknown")$ ]] && { echo "Applying workaround for dnsmasq bug (compare issue #1446)" service systemd-resolved stop || true service dnsmasq start @@ -24,6 +24,7 @@ install() } service dnsmasq stop + [[ "$INIT_SYSTEM" == "systemd" ]] && service systemd-resolved start || true update-rc.d dnsmasq disable || rm /etc/systemd/system/multi-user.target.wants/dnsmasq.service [[ "$DOCKERBUILD" == 1 ]] && { diff --git a/build/build-LXD.sh b/build/build-LXD.sh index 5793a26e..bf639347 100755 --- a/build/build-LXD.sh +++ b/build/build-LXD.sh @@ -8,7 +8,7 @@ # Usage: # -set -e +set -ex source build/buildlib.sh echo -e "\e[1m\n[ Build NCP LXD ]\e[0m" @@ -32,24 +32,26 @@ prepare_dirs # tmp cache output ## BUILD NCP lxc delete -f ncp 2>/dev/null || true -systemd-run --user --scope -p "Delegate=yes" lxc launch images:debian/bullseye ncp +systemd-run --user --scope -p "Delegate=yes" lxc launch -q images:debian/bullseye ncp lxc config device add ncp buildcode disk source="$(pwd)" path=/build lxc exec ncp -- bash -c 'while [ "$(systemctl is-system-running 2>/dev/null)" != "running" ] && [ "$(systemctl is-system-running 2>/dev/null)" != "degraded" ]; do :; done' -lxc exec ncp -- bash -c 'CODE_DIR=/build bash /build/install.sh' +lxc exec ncp -- bash -c 'CODE_DIR=/build DBG=x bash /build/install.sh' lxc exec ncp -- bash -c 'source /build/etc/library.sh; run_app_unsafe /build/post-inst.sh' lxc stop ncp lxc config device remove ncp buildcode -lxc publish ncp -f --alias ncp/"${version}" +lxc publish -q ncp -f --alias ncp/"${version}" ## pack -lxc export ncp "$TAR" +[[ " $* " =~ .*" --pack ".* ]] && lxc image export -q ncp/"${version}" "$TAR" + +exit 0 ## test #set_static_IP "$IMG" "$IP" #test_image "$IMG" "$IP" # upload -create_torrent "$TAR" +#create_torrent "$TAR" #upload_ftp "$( basename "$TAR" .tar.bz2 )" diff --git a/build/build-SD-armbian.sh b/build/build-SD-armbian.sh index ad72c790..a7e3977d 100755 --- a/build/build-SD-armbian.sh +++ b/build/build-SD-armbian.sh @@ -8,7 +8,7 @@ # Usage: ./build-SD-armbian.sh [] # -set -e +set -ex source build/buildlib.sh #CLEAN=0 # Pass this envvar to avoid cleaning download cache @@ -67,16 +67,21 @@ EXTRA_CONF=build/armbian/"config-$BOARD".conf # build rm -rf armbian/output/images +mkdir -p armbian/userpatches +sed -e '/docker.*run/s/-it//' armbian/config/templates/config-docker.conf > armbian/userpatches/config-docker.conf +docker pull "ghcr.io/armbian/build:$(cut -d"." -f1-2 < armbian/VERSION)-$(dpkg --print-architecture)" armbian/compile.sh docker ncp rm "$CONF" # pack image -mv armbian/output/images/Armbian*.img "$IMG" -pack_image "$IMG" "$TAR" + +[[ " $* " =~ " --pack " ]] && { mv armbian/output/images/Armbian*.img "$IMG" && pack_image "$IMG" "$TAR"; } + +exit 0 # test # TODO # upload -create_torrent "$TAR" +#create_torrent "$TAR" #upload_ftp "$( basename "$TAR" .tar.bz2 )" diff --git a/build/build-SD-rpi.sh b/build/build-SD-rpi.sh index a0a4aafd..223aed65 100755 --- a/build/build-SD-rpi.sh +++ b/build/build-SD-rpi.sh @@ -96,7 +96,9 @@ trap '' EXIT clean_chroot_raspbian ## pack -pack_image "$IMG" "$TAR" +[[ "$*" =~ .*" --pack ".* ]] && pack_image "$IMG" "$TAR" + +exit 0 ## test @@ -104,7 +106,7 @@ pack_image "$IMG" "$TAR" #test_image "$IMG" "$IP" # TODO fix tests # upload -create_torrent "$TAR" +#create_torrent "$TAR" #upload_ftp "$( basename "$TAR" .tar.bz2 )" diff --git a/build/buildlib.sh b/build/buildlib.sh index fb6be298..bef36cf0 100644 --- a/build/buildlib.sh +++ b/build/buildlib.sh @@ -173,7 +173,12 @@ function prepare_chroot_raspbian() sudo mount -o bind /dev raspbian_root/dev/ sudo mount -o bind /dev/pts raspbian_root/dev/pts - sudo cp /usr/bin/qemu-aarch64-static raspbian_root/usr/bin + if [[ -f "qemu-aarch64-static" ]] + then + sudo cp qemu-aarch64-static raspbian_root/usr/bin/ + else + sudo cp /usr/bin/qemu-aarch64-static raspbian_root/usr/bin + fi # Prevent services from auto-starting sudo bash -c "echo -e '#!/bin/sh\nexit 101' > raspbian_root/usr/sbin/policy-rc.d" @@ -302,7 +307,7 @@ function pack_image() local IMGNAME="$( basename "$IMG" )" echo -e "\n\e[1m[ Pack Image ]\e[0m" echo "packing $IMG → $TAR" - tar -I pbzip2 -C "$DIR" -cvf "$TAR" "$IMGNAME" && \ + tar -C "$DIR" -cavf "$TAR" "$IMGNAME" && \ echo -e "$TAR packed successfully" } diff --git a/tests/nextcloud_tests.py b/tests/nextcloud_tests.py index 1712e67a..6005595d 100755 --- a/tests/nextcloud_tests.py +++ b/tests/nextcloud_tests.py @@ -46,6 +46,10 @@ class tc: normal='\033[0m' +class TestFailed(Exception): + pass + + class Test: title = "test" @@ -53,18 +57,24 @@ class Test: self.title = title print("[check] " + "{:16}".format(title), end=' ', flush = True) - def check(self, expression): + def check(self, expression, msg=None): if expression: print(tc.green + "ok" + tc.normal) self.log("ok") else: print(tc.red + "error" + tc.normal) self.log("error") - sys.exit(1) + exc_args = [f"'{self.title}' failed"] + if isinstance(expression, Exception): + exc_args.append(expression) + if msg is not None: + exc_args.append(msg) - def report(self, title, expression): + raise TestFailed(*exc_args) + + def report(self, title, expression, msg=None): self.new(title) - self.check(expression) + self.check(expression, msg=msg) def log(self, result): config = configparser.ConfigParser() @@ -112,13 +122,11 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver): test.new("nextcloud page") try: driver.get(f"https://{IP}:{nc_port}/index.php/settings/admin/overview") - except: - test.check(False) - print(tc.red + "error:" + tc.normal + " unable to reach " + tc.yellow + IP + tc.normal) - sys.exit(1) - test.check("NextCloudPi" in driver.title) + except Exception as e: + test.check(e, msg=f"{tc.red}error:{tc.normal} unable to reach {tc.yellow + IP + tc.normal}") + test.check("NextCloudPi" in driver.title, msg="NextCloudPi not found in page title!") trusted_domain_str = "You are accessing the server from an untrusted domain" - test.report("trusted domain", trusted_domain_str not in driver.page_source) + test.report("trusted domain", trusted_domain_str not in driver.page_source, f"Domain '{IP}' is not trusted") try: driver.find_element(By.ID, "user").send_keys(nc_user) driver.find_element(By.ID, "password").send_keys(nc_pass) @@ -129,7 +137,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver): except NoSuchElementException: pass - test.report("password", "Wrong password" not in driver.page_source) + test.report("password", "Wrong password" not in driver.page_source, msg="Failed to login with provided password") test.new("settings config") wait = WebDriverWait(driver, 30) @@ -167,9 +175,7 @@ def test_nextcloud(IP: str, nc_port: str, driver: WebDriver): test.check(True) except Exception as e: - test.check(False) - print(e) - print(traceback.format_exc()) + test.check(e) if __name__ == "__main__": @@ -230,6 +236,9 @@ if __name__ == "__main__": driver = webdriver.Firefox(service_log_path='/dev/null', options=options) try: test_nextcloud(IP, nc_port, driver) + except Exception as e: + print(e) + print(traceback.format_exc()) finally: driver.close() -- cgit v1.2.3