From 1a80ecbfc900ee1e241e9332aa0cbab749c3b5ec Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 13:43:12 +0200 Subject: 90-ncp.cnf.sh: Don't use /data-ro for database path Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- etc/ncp-templates/mysql/90-ncp.cnf.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etc/ncp-templates/mysql/90-ncp.cnf.sh b/etc/ncp-templates/mysql/90-ncp.cnf.sh index 6b9efe9f..dccb2597 100644 --- a/etc/ncp-templates/mysql/90-ncp.cnf.sh +++ b/etc/ncp-templates/mysql/90-ncp.cnf.sh @@ -11,7 +11,7 @@ else if [[ "$DOCKERBUILD" -eq 1 ]] then echo "INFO: Docker build detected." >&2 - DB_DIR=/data-ro/database + DB_DIR=/data/database elif is_docker then echo "INFO: Docker container detected." >&2 -- cgit v1.2.3 From 412efad20f27c7af17473615d60deefad3bbcdfa Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 14:20:14 +0200 Subject: run-parts.sh: Backup docker instance when starting container Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/debian-ncp/run-parts.sh | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index 0099a5a1..abcde008 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -13,7 +13,31 @@ sed -i 's|/data-ro|/data|' "/etc/mysql/mariadb.conf.d/90-ncp.cnf" || true trap cleanup SIGTERM # if an empty volume is mounted to /data, pre-populate it -[[ $( ls -1A /data | wc -l ) -eq 0 ]] && { echo "Initializing empty volume.."; cp -raT /data-ro /data; } +if [[ $( ls -1A /data | wc -l ) -eq 0 ]] +then + echo "Initializing empty volume.." + cp -raT /data-ro /data +else + echo "Cleanup old startup backups..." + BKPS="$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null)" + while read -r bkp + do + rm -f "$BKPDIR/$bkp" + done <"$(echo "$BKPS" | tail -n + 5)" + BKPDIR=/data/ncp-startup-backups/ + WITH_DATA=no + COMPRESSED=yes + LIMIT=0 + mkdir -p "$BKPDIR" + echo "Back up current instance..." + set +eE + if ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" + then + echo "Backup stored at '$BKPDIR/$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null | head -n1)'" + else + echo 'WARN: Backup creation failed' + fi +fi # wrapper to simulate update-rc.d cat > /usr/local/sbin/update-rc.d <<'EOF' -- cgit v1.2.3 From 4049a4b07d5009e4e6e23fe8fde2ee29e5fae08f Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 14:55:17 +0200 Subject: 90-ncp.cnf.sh: Fix /data-ro being used as db directory outside of installation Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- etc/ncp-templates/mysql/90-ncp.cnf.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etc/ncp-templates/mysql/90-ncp.cnf.sh b/etc/ncp-templates/mysql/90-ncp.cnf.sh index dccb2597..c61fd1bc 100644 --- a/etc/ncp-templates/mysql/90-ncp.cnf.sh +++ b/etc/ncp-templates/mysql/90-ncp.cnf.sh @@ -8,10 +8,10 @@ then echo "INFO: Restoring template to default settings" >&2 DB_DIR=/var/lib/mysql else - if [[ "$DOCKERBUILD" -eq 1 ]] + if is_docker && [[ -f /.ncp-image ]] then echo "INFO: Docker build detected." >&2 - DB_DIR=/data/database + DB_DIR=/data-ro/database elif is_docker then echo "INFO: Docker container detected." >&2 -- cgit v1.2.3 From 549ba6aad3644b396eff2f4b9a3a6ca8e6f2968b Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 15:03:56 +0200 Subject: Dockerfile: Mark as image build during lamp setup Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/Dockerfile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 438009b5..557b86f3 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -35,6 +35,9 @@ COPY etc/ncp-templates /usr/local/etc/ncp-templates RUN --mount=type=cache,target=/var/cache/apt --mount=type=cache,target=/var/lib/apt \ set -e; \ +# mark as image build +touch /.ncp-image; \ + # installation apt-get update; \ apt-get install --no-install-recommends -y jq wget procps ca-certificates lsb-release; \ @@ -62,7 +65,8 @@ rm -f /var/log/alternatives.log /var/log/apt/*; \ # specific cleanup rm /data-ro/database/ib_logfile*; \ -rm /usr/local/etc/lamp.sh +rm /usr/local/etc/lamp.sh \ +rm /.ncp-image; COPY build/docker/lamp/010lamp /etc/services-enabled.d/ -- cgit v1.2.3 From 879a44ba9c7614fd8a7d13f8cb878336f32d017c Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 15:09:31 +0200 Subject: Dockerfile: Add missing semicolon Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 557b86f3..4ee3d164 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -65,7 +65,7 @@ rm -f /var/log/alternatives.log /var/log/apt/*; \ # specific cleanup rm /data-ro/database/ib_logfile*; \ -rm /usr/local/etc/lamp.sh \ +rm /usr/local/etc/lamp.sh; \ rm /.ncp-image; COPY build/docker/lamp/010lamp /etc/services-enabled.d/ -- cgit v1.2.3 From 95411b58e14d5935a949e12d4fce894d73ea0dae Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 15:16:23 +0200 Subject: Dockerfile: Fix db directory after installation Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/build/docker/Dockerfile b/build/docker/Dockerfile index 4ee3d164..1f813bec 100644 --- a/build/docker/Dockerfile +++ b/build/docker/Dockerfile @@ -172,6 +172,7 @@ sed -i 's|data-ro|data|' /data-ro/nextcloud/config/config.php; \ # fix default paths sed -i 's|/media/USBdrive|/data/backups|' /usr/local/etc/ncp-config.d/nc-backup.cfg; \ sed -i 's|/media/USBdrive|/data/backups|' /usr/local/etc/ncp-config.d/nc-backup-auto.cfg; \ +sed -i 's|/data-ro|/data|' /etc/mysql/mariadb.conf.d/90-ncp.cnf; \ # cleanup all NCP extras run_app_unsafe post-inst.sh; \ -- cgit v1.2.3 From 1c4fefbb7af9e775a45e450e5e7378ea07f74b5f Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 15:22:22 +0200 Subject: run-parts.sh: Fix syntax Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/debian-ncp/run-parts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index abcde008..0bdade4b 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -23,7 +23,7 @@ else while read -r bkp do rm -f "$BKPDIR/$bkp" - done <"$(echo "$BKPS" | tail -n + 5)" + done <"$(echo "$BKPS" | tail -n +5)" BKPDIR=/data/ncp-startup-backups/ WITH_DATA=no COMPRESSED=yes -- cgit v1.2.3 From 9b5fbba51776017d9b9b12de3ad93a86aa4c51c7 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 15:31:45 +0200 Subject: run-parts.sh: Fix undefined backup location Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/debian-ncp/run-parts.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index 0bdade4b..4a01146e 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -17,18 +17,19 @@ if [[ $( ls -1A /data | wc -l ) -eq 0 ]] then echo "Initializing empty volume.." cp -raT /data-ro /data -else - echo "Cleanup old startup backups..." - BKPS="$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null)" - while read -r bkp - do - rm -f "$BKPDIR/$bkp" - done <"$(echo "$BKPS" | tail -n +5)" +elif [[ -z "$NOBACKUP" ]] || [[ "$NOBACKUP" != "true" ]] +then BKPDIR=/data/ncp-startup-backups/ WITH_DATA=no COMPRESSED=yes LIMIT=0 mkdir -p "$BKPDIR" + echo "Cleanup old startup backups..." + BKPS="$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null || true)" + while read -r bkp + do + rm -f "$BKPDIR/$bkp" + done <"$(echo "$BKPS" | tail -n +5)" echo "Back up current instance..." set +eE if ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" -- cgit v1.2.3 From 6b8501e3aa36e9ef15e3c8abad91c7868fc425ad Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 15:43:28 +0200 Subject: run-parts.sh: Enable debug output during backups Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/debian-ncp/run-parts.sh | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index 4a01146e..0bbfc28a 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -19,17 +19,22 @@ then cp -raT /data-ro /data elif [[ -z "$NOBACKUP" ]] || [[ "$NOBACKUP" != "true" ]] then + set -x BKPDIR=/data/ncp-startup-backups/ WITH_DATA=no COMPRESSED=yes LIMIT=0 mkdir -p "$BKPDIR" echo "Cleanup old startup backups..." - BKPS="$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null || true)" - while read -r bkp - do - rm -f "$BKPDIR/$bkp" - done <"$(echo "$BKPS" | tail -n +5)" + skip_bkp_cleanup=0 + BKPS="$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null || skip_bkp_cleanup=1)" + if [[ "$skip_bkp_cleanup" == 0 ]] + then + while read -r bkp + do + rm -f "$BKPDIR/$bkp" + done <"$(echo "$BKPS" | tail -n +5)" + fi echo "Back up current instance..." set +eE if ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" @@ -38,6 +43,7 @@ then else echo 'WARN: Backup creation failed' fi + set +x fi # wrapper to simulate update-rc.d -- cgit v1.2.3 From 1499cfbdc28bb2126825d16bc23b8ab01cfe3d85 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 16:00:12 +0200 Subject: run-parts.sh: Do the backup on shutdown not on startup Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/debian-ncp/run-parts.sh | 54 ++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index 0bbfc28a..cae28904 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -5,6 +5,33 @@ cleanup() for file in $( ls -1rv /etc/services-enabled.d ); do /etc/services-enabled.d/"$file" stop "$1" done + if [[ -z "$NOBACKUP" ]] || [[ "$NOBACKUP" != "true" ]] + then + BKPDIR=/data/docker-shutdown-backups/ + WITH_DATA=no + COMPRESSED=yes + LIMIT=0 + mkdir -p "$BKPDIR" + echo "Cleanup old shutdown backups..." + skip_bkp_cleanup=0 + BKPS="$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null || skip_bkp_cleanup=1)" + if [[ "$skip_bkp_cleanup" == 0 ]] + then + while read -r bkp + do + rm -f "$BKPDIR/$bkp" + done <"$(echo "$BKPS" | tail -n +5)" + fi + echo "Back up current instance..." + set +eE + if ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" + then + echo "Backup stored at '$BKPDIR/$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null | head -n1)'" + else + echo 'WARN: Backup creation failed' + fi + set +x + fi exit } @@ -17,33 +44,6 @@ if [[ $( ls -1A /data | wc -l ) -eq 0 ]] then echo "Initializing empty volume.." cp -raT /data-ro /data -elif [[ -z "$NOBACKUP" ]] || [[ "$NOBACKUP" != "true" ]] -then - set -x - BKPDIR=/data/ncp-startup-backups/ - WITH_DATA=no - COMPRESSED=yes - LIMIT=0 - mkdir -p "$BKPDIR" - echo "Cleanup old startup backups..." - skip_bkp_cleanup=0 - BKPS="$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null || skip_bkp_cleanup=1)" - if [[ "$skip_bkp_cleanup" == 0 ]] - then - while read -r bkp - do - rm -f "$BKPDIR/$bkp" - done <"$(echo "$BKPS" | tail -n +5)" - fi - echo "Back up current instance..." - set +eE - if ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" - then - echo "Backup stored at '$BKPDIR/$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null | head -n1)'" - else - echo 'WARN: Backup creation failed' - fi - set +x fi # wrapper to simulate update-rc.d -- cgit v1.2.3 From ba7b845c73410ed3717a00acd42b3ea7621a1f01 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 16:17:59 +0200 Subject: run-parts.sh: Add information about critical bug Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/debian-ncp/run-parts.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index cae28904..2b779771 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -35,6 +35,10 @@ cleanup() exit } +grep '/data-ro' /etc/mysql/mariadb.conf.d/90-ncp.cnf > /dev/null 2>&1 \ + && echo "WARNING: Looks like you have been affected by a critical bug in NCP that can cause data loss. We're trying" \ + "to fix this now, but if you encounter any issues, please check" \ + "https://github.com/nextcloud/nextcloudpi/issues/1577#issuecomment-1260830341" sed -i 's|/data-ro|/data|' "/etc/mysql/mariadb.conf.d/90-ncp.cnf" || true trap cleanup SIGTERM -- cgit v1.2.3 From fc79b722f227af156d86c670c7072119fb252fbd Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 19:37:40 +0200 Subject: run-parts.sh: Fix wrong permissions on /data/database Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/debian-ncp/run-parts.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index 2b779771..ee254449 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -30,15 +30,16 @@ cleanup() else echo 'WARN: Backup creation failed' fi - set +x fi exit } -grep '/data-ro' /etc/mysql/mariadb.conf.d/90-ncp.cnf > /dev/null 2>&1 \ - && echo "WARNING: Looks like you have been affected by a critical bug in NCP that can cause data loss. We're trying" \ +grep '/data-ro' /etc/mysql/mariadb.conf.d/90-ncp.cnf > /dev/null 2>&1 && { + echo "WARNING: Looks like you have been affected by a critical bug in NCP that can cause data loss. We're trying" \ "to fix this now, but if you encounter any issues, please check" \ "https://github.com/nextcloud/nextcloudpi/issues/1577#issuecomment-1260830341" + chown -R mysql: /data/database || true +} sed -i 's|/data-ro|/data|' "/etc/mysql/mariadb.conf.d/90-ncp.cnf" || true trap cleanup SIGTERM @@ -87,3 +88,5 @@ done # wait for trap from 'docker stop' echo "Init done" while true; do sleep 0.5; done + +} -- cgit v1.2.3 From a6aaa4abe8a2df9d8b3255eaae5ba16d1a616ccf Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 20:34:26 +0200 Subject: lamp.sh: Create systemd users manually, prevent changed uids build-docker.yml: Add update tests Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 116 +++++++++++++++++++++++++++++++++++ build/docker/debian-ncp/run-parts.sh | 3 +- lamp.sh | 6 ++ 3 files changed, 124 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 0a9b91a0..bae34c1d 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -61,6 +61,122 @@ jobs: docker push "thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}" docker push "thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}-testing" + update-from-latest: + needs: + - build + runs-on: ubuntu-latest + env: + VERSION: "${{ inputs.git_ref || github.ref }}" + + strategy: + matrix: + arch: + - armhf + - x86 + - arm64 + fail-fast: false + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Start ncp container + run: | + docker volume create ncdata + docker run --platform "linux/${{ matrix.arch }}" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ownyourbits/nextcloudpi:latest + + - name: Checkout code + uses: actions/checkout@v3 + with: + ref: "${{ env.VERSION }}" + + - name: Setup Firefox + uses: browser-actions/setup-firefox@latest + - name: Setup GeckoDriver + uses: browser-actions/setup-geckodriver@latest + - name: Setup Selenium + run: pip install selenium + + - name: Activate container + working-directory: ./tests + run: | + docker logs nextcloudpi |& awk '{ print "CONTAINER::" $0 }' + echo 'CONTAINER:: ==========' + docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & + docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + cmd=(python activation_tests.py --no-gui localhost 8443 4443) + [[ "${{ matrix.arch }}" == "x86" ]] || cmd+=(--timeout 300) + + success=false + for attempt in {1..3} + do + echo ":: Activation Tests (attempt $attempt/3) ::" + "${cmd[@]}" || { + tail -n 20 geckodriver.log >&2 || true + echo "CI:: Activation test (attempt $attempt/3) failed!" + docker exec nextcloudpi bash /usr/local/bin/ncp-diag + sleep 12 + continue + } + success=true + break + done + [[ "$success" == "true" ]] || { + echo "CI:: Activation test failed in all attempts!" + exit 1 + } + echo "CI:: Activation test successful" + - name: Start new container + run: | + docker stop nextcloudpi + docker run -d -v ncdata:/data --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}-testing + - name: Integration Tests + working-directory: ./tests + run: | + docker logs nextcloudpi |& awk '{ print "CONTAINER::" $0 }' + echo 'CONTAINER:: ==========' + docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & + docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & + cmd=(python activation_tests.py --no-gui localhost 8443 4443) + [[ "${{ matrix.arch }}" == "x86" ]] || cmd+=(--timeout 300) + + success=false + for attempt in {1..3} + do + echo ":: System Tests (attempt $attempt/3) ::" + python system_tests.py --no-ping --non-interactive || { + echo "CI:: System test (attempt $attempt) failed!" + continue + } + success=true + break + done + [[ "$success" == "true" ]] || { + echo "CI:: System test failed in all attempts!" + exit 1 + } + echo "CI:: System test successful" + + success=false + for attempt in {1..3} + do + echo ":: Nextcloud Tests (attempt $attempt/3) ::" + python nextcloud_tests.py --no-gui localhost 8443 4443 || { + tail -n 20 geckodriver.log >&2 || true + echo "CI:: Nextcloud test (attempt $attempt/3) failed!" + continue + } + success=true + break + done + [[ "$success" == "true" ]] || { + echo "CI:: Nextcloud test failed in all attempts!" + exit 1 + } + echo "CI:: Nextcloud test successful" + + + + test: needs: - build diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index ee254449..15516e7e 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -37,7 +37,8 @@ cleanup() grep '/data-ro' /etc/mysql/mariadb.conf.d/90-ncp.cnf > /dev/null 2>&1 && { echo "WARNING: Looks like you have been affected by a critical bug in NCP that can cause data loss. We're trying" \ "to fix this now, but if you encounter any issues, please check" \ - "https://github.com/nextcloud/nextcloudpi/issues/1577#issuecomment-1260830341" + "https://github.com/nextcloud/nextcloudpi/issues/1577#issuecomment-1260830341" \ + "It is likely that you will have to restore a backup" chown -R mysql: /data/database || true } sed -i 's|/data-ro|/data|' "/etc/mysql/mariadb.conf.d/90-ncp.cnf" || true diff --git a/lamp.sh b/lamp.sh index d535fa9b..00df8d14 100644 --- a/lamp.sh +++ b/lamp.sh @@ -36,6 +36,12 @@ install() mkdir -p /run/lock apache2ctl -V || true + # Create systemd users to keep uids persistent between containers + addgroup --quiet --system systemd-journal + adduser --quiet -u 180 --system --group --no-create-home --home /run/systemd \ + --gecos "systemd Network Management" systemd-network + adduser --quiet -u 181 --system --group --no-create-home --home /run/systemd \ + --gecos "systemd Resolver" systemd-resolve install_with_shadow_workaround --no-install-recommends systemd $APTINSTALL -t $RELEASE php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \ php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \ -- cgit v1.2.3 From 6906c5072974a836abfad4fa6315de67625950c4 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 20:40:19 +0200 Subject: build-docker.yml: Use inputs instead of matrix strategy for speeding up builds Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 55 ++++++++++++-------------------------- .github/workflows/release.yml | 19 ++++++++++++- 2 files changed, 35 insertions(+), 39 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index bae34c1d..726ed29b 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -10,6 +10,9 @@ on: git_ref: required: true type: string + arch: + required: true + type: string push: branches: - master @@ -23,13 +26,6 @@ jobs: build: if: ${{ github.repository == 'nextcloud/nextcloudpi' }} runs-on: ubuntu-latest - strategy: - matrix: - arch: - - x86 - - armhf - - arm64 - fail-fast: false env: VERSION: "${{ inputs.git_ref || github.ref }}" steps: @@ -53,28 +49,21 @@ jobs: - name: Build images id: build-container run: | - ./build/build-docker.sh "${{ matrix.arch }}" - docker tag "ownyourbits/nextcloudpi-${{ matrix.arch }}:latest" "thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}" - testing_image="ownyourbits/nextcloudpi-${{ matrix.arch }}:latest" - [[ "${{ matrix.arch }}" == "arm64" ]] && testing_image="ownyourbits/ncp-qemu-fix-${{ matrix.arch }}:latest" - docker tag "${testing_image}" "thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}-testing" - docker push "thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}" - docker push "thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}-testing" + ./build/build-docker.sh "${{ inputs.arch }}" + docker tag "ownyourbits/nextcloudpi-${{ inputs.arch }}:latest" "thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}" + testing_image="ownyourbits/nextcloudpi-${{ inputs.arch }}:latest" + [[ "${{ inputs.arch }}" == "arm64" ]] && testing_image="ownyourbits/ncp-qemu-fix-${{ inputs.arch }}:latest" + docker tag "${testing_image}" "thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}-testing" + docker push "thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}" + docker push "thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}-testing" - update-from-latest: + update-test: needs: - build runs-on: ubuntu-latest env: VERSION: "${{ inputs.git_ref || github.ref }}" - strategy: - matrix: - arch: - - armhf - - x86 - - arm64 - fail-fast: false steps: - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -82,7 +71,7 @@ jobs: - name: Start ncp container run: | docker volume create ncdata - docker run --platform "linux/${{ matrix.arch }}" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ownyourbits/nextcloudpi:latest + docker run --platform "linux/${{ inputs.arch }}" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ownyourbits/nextcloudpi:latest - name: Checkout code uses: actions/checkout@v3 @@ -104,7 +93,7 @@ jobs: docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & cmd=(python activation_tests.py --no-gui localhost 8443 4443) - [[ "${{ matrix.arch }}" == "x86" ]] || cmd+=(--timeout 300) + [[ "${{ inputs.arch }}" == "x86" ]] || cmd+=(--timeout 300) success=false for attempt in {1..3} @@ -128,7 +117,7 @@ jobs: - name: Start new container run: | docker stop nextcloudpi - docker run -d -v ncdata:/data --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}-testing + docker run -d -v ncdata:/data --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}-testing - name: Integration Tests working-directory: ./tests run: | @@ -137,7 +126,7 @@ jobs: docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & cmd=(python activation_tests.py --no-gui localhost 8443 4443) - [[ "${{ matrix.arch }}" == "x86" ]] || cmd+=(--timeout 300) + [[ "${{ inputs.arch }}" == "x86" ]] || cmd+=(--timeout 300) success=false for attempt in {1..3} @@ -173,9 +162,6 @@ jobs: exit 1 } echo "CI:: Nextcloud test successful" - - - test: needs: @@ -184,13 +170,6 @@ jobs: env: VERSION: "${{ inputs.git_ref || github.ref }}" - strategy: - matrix: - arch: - - armhf - - x86 - - arm64 - fail-fast: false steps: - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -200,7 +179,7 @@ jobs: echo "${{ secrets.DOCKER_PASSWORD_INTERNAL }}" | docker login -u "${{ secrets.DOCKER_LOGIN_INTERNAL }}" --password-stdin - name: Start ncp container run: | - docker run -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${{ matrix.arch }}:${{ github.run_id }}-testing + docker run -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}-testing - name: Checkout code uses: actions/checkout@v3 @@ -222,7 +201,7 @@ jobs: docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & cmd=(python activation_tests.py --no-gui localhost 8443 4443) - [[ "${{ matrix.arch }}" == "x86" ]] || cmd+=(--timeout 300) + [[ "${{ inputs.arch }}" == "x86" ]] || cmd+=(--timeout 300) success=false for attempt in {1..3} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6ddf9f15..f950991f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -47,11 +47,28 @@ jobs: git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" secrets: inherit - docker: + docker-x86: if: ${{ inputs.docker || github.event_name != 'workflow_dispatch' }} uses: ./.github/workflows/build-docker.yml with: git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + arch: "x86" + secrets: inherit + + docker-armhf: + if: ${{ inputs.docker || github.event_name != 'workflow_dispatch' }} + uses: ./.github/workflows/build-docker.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + arch: "armhf" + secrets: inherit + + docker-arm64: + if: ${{ inputs.docker || github.event_name != 'workflow_dispatch' }} + uses: ./.github/workflows/build-docker.yml + with: + git_ref: "${{ inputs.git_ref || github.head_ref || github.ref_name }}" + arch: "arm64" secrets: inherit test-curl-installer: -- cgit v1.2.3 From e310f33663fc9978f4be60f8cb715e51a24389b3 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 20:41:20 +0200 Subject: build-docker.yml: Add default arch Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 726ed29b..b840c763 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -11,8 +11,9 @@ on: required: true type: string arch: - required: true + required: false type: string + default: 'x86' push: branches: - master -- cgit v1.2.3 From a226b53da0a8132ef3699f15d487ef2e7d0707ab Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 20:45:13 +0200 Subject: build-docker.yml: Use env variable for arch detection Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index b840c763..63ede1b9 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -29,6 +29,7 @@ jobs: runs-on: ubuntu-latest env: VERSION: "${{ inputs.git_ref || github.ref }}" + ARCH: "${{ inputs.arch }}" steps: - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -50,13 +51,13 @@ jobs: - name: Build images id: build-container run: | - ./build/build-docker.sh "${{ inputs.arch }}" - docker tag "ownyourbits/nextcloudpi-${{ inputs.arch }}:latest" "thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}" - testing_image="ownyourbits/nextcloudpi-${{ inputs.arch }}:latest" - [[ "${{ inputs.arch }}" == "arm64" ]] && testing_image="ownyourbits/ncp-qemu-fix-${{ inputs.arch }}:latest" - docker tag "${testing_image}" "thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}-testing" - docker push "thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}" - docker push "thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}-testing" + ./build/build-docker.sh "${ARCH?}" + docker tag "ownyourbits/nextcloudpi-${ARCH?}:latest" "thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}" + testing_image="ownyourbits/nextcloudpi-${ARCH?}:latest" + [[ "${ARCH?}" == "arm64" ]] && testing_image="ownyourbits/ncp-qemu-fix-${ARCH?}:latest" + docker tag "${testing_image}" "thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing" + docker push "thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}" + docker push "thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing" update-test: needs: @@ -64,7 +65,7 @@ jobs: runs-on: ubuntu-latest env: VERSION: "${{ inputs.git_ref || github.ref }}" - + ARCH: "${{ inputs.arch }}" steps: - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -72,7 +73,7 @@ jobs: - name: Start ncp container run: | docker volume create ncdata - docker run --platform "linux/${{ inputs.arch }}" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ownyourbits/nextcloudpi:latest + docker run --platform "linux/${ARCH?}" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ownyourbits/nextcloudpi:latest - name: Checkout code uses: actions/checkout@v3 @@ -94,7 +95,7 @@ jobs: docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & cmd=(python activation_tests.py --no-gui localhost 8443 4443) - [[ "${{ inputs.arch }}" == "x86" ]] || cmd+=(--timeout 300) + [[ "${ARCH?}" == "x86" ]] || cmd+=(--timeout 300) success=false for attempt in {1..3} @@ -118,7 +119,7 @@ jobs: - name: Start new container run: | docker stop nextcloudpi - docker run -d -v ncdata:/data --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}-testing + docker run -d -v ncdata:/data --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing - name: Integration Tests working-directory: ./tests run: | @@ -127,7 +128,7 @@ jobs: docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & cmd=(python activation_tests.py --no-gui localhost 8443 4443) - [[ "${{ inputs.arch }}" == "x86" ]] || cmd+=(--timeout 300) + [[ "${ARCH?}" == "x86" ]] || cmd+=(--timeout 300) success=false for attempt in {1..3} @@ -170,7 +171,7 @@ jobs: runs-on: ubuntu-latest env: VERSION: "${{ inputs.git_ref || github.ref }}" - + ARCH: "${{ inputs.arch }}" steps: - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -180,7 +181,7 @@ jobs: echo "${{ secrets.DOCKER_PASSWORD_INTERNAL }}" | docker login -u "${{ secrets.DOCKER_LOGIN_INTERNAL }}" --password-stdin - name: Start ncp container run: | - docker run -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${{ inputs.arch }}:${{ github.run_id }}-testing + docker run -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi thecalcaholic/ncp-internal-${ARCH?}:${{ github.run_id }}-testing - name: Checkout code uses: actions/checkout@v3 @@ -202,7 +203,7 @@ jobs: docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & cmd=(python activation_tests.py --no-gui localhost 8443 4443) - [[ "${{ inputs.arch }}" == "x86" ]] || cmd+=(--timeout 300) + [[ "${ARCH?}" == "x86" ]] || cmd+=(--timeout 300) success=false for attempt in {1..3} -- cgit v1.2.3 From eea991fc49959f5f7986b3fab1911336e9d3d33a Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 20:47:50 +0200 Subject: build-docker.yml: Add fallback value for arch Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 63ede1b9..fa46079d 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -29,7 +29,7 @@ jobs: runs-on: ubuntu-latest env: VERSION: "${{ inputs.git_ref || github.ref }}" - ARCH: "${{ inputs.arch }}" + ARCH: "${{ inputs.arch || 'x86' }}" steps: - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -65,7 +65,7 @@ jobs: runs-on: ubuntu-latest env: VERSION: "${{ inputs.git_ref || github.ref }}" - ARCH: "${{ inputs.arch }}" + ARCH: "${{ inputs.arch || 'x86' }}" steps: - name: Set up QEMU uses: docker/setup-qemu-action@v1 @@ -171,7 +171,7 @@ jobs: runs-on: ubuntu-latest env: VERSION: "${{ inputs.git_ref || github.ref }}" - ARCH: "${{ inputs.arch }}" + ARCH: "${{ inputs.arch || 'x86' }}" steps: - name: Set up QEMU uses: docker/setup-qemu-action@v1 -- cgit v1.2.3 From c8aad36199a262ac0733ae15402ee122d76a49ea Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 20:57:05 +0200 Subject: build-docker.yml: Fix docker manifest platform Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index fa46079d..ad7ca039 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -73,7 +73,9 @@ jobs: - name: Start ncp container run: | docker volume create ncdata - docker run --platform "linux/${ARCH?}" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ownyourbits/nextcloudpi:latest + platform=linux/${ARCH?} + [[ "$ARCH" == 'x86' ]] && platform=linux/amd64 + docker run --platform "$platform" -v ncdata:/data -d --rm -p 8443:443 -p 4443:4443 --name nextcloudpi ownyourbits/nextcloudpi:latest - name: Checkout code uses: actions/checkout@v3 -- cgit v1.2.3 From c75f100b55e56752e57e0c6dc948bf499bfffc1a Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 21:07:03 +0200 Subject: build-docker.yml: Add timeouts to update test Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index ad7ca039..f7e2bebd 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -131,6 +131,8 @@ jobs: docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & cmd=(python activation_tests.py --no-gui localhost 8443 4443) [[ "${ARCH?}" == "x86" ]] || cmd+=(--timeout 300) + + sleep 20 success=false for attempt in {1..3} @@ -138,6 +140,7 @@ jobs: echo ":: System Tests (attempt $attempt/3) ::" python system_tests.py --no-ping --non-interactive || { echo "CI:: System test (attempt $attempt) failed!" + sleep 12 continue } success=true @@ -156,6 +159,7 @@ jobs: python nextcloud_tests.py --no-gui localhost 8443 4443 || { tail -n 20 geckodriver.log >&2 || true echo "CI:: Nextcloud test (attempt $attempt/3) failed!" + sleep 12 continue } success=true -- cgit v1.2.3 From c8eb8a49b5a59bcbbc2c11816ff0b63ef7c24b01 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 21:12:01 +0200 Subject: release.yml: Fix dependencies of docker-release job Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/release.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f950991f..69609cd5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -258,7 +258,9 @@ jobs: docker-release: needs: - - docker + - docker-x86 + - docker-armhf + - docker-arm64 - test-curl-installer if: ${{ (inputs.release || github.event_name != 'workflow_dispatch') && github.ref_type == 'tag' && (github.ref_protected || startsWith(github.ref, 'refs/tags/v')) }} runs-on: ubuntu-latest -- cgit v1.2.3 From ff90eedf92935488d45c23b55cb6d2249dcf5e4f Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 21:17:23 +0200 Subject: build-docker.yml: Add status information Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index f7e2bebd..a7c4d9bf 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -170,6 +170,8 @@ jobs: exit 1 } echo "CI:: Nextcloud test successful" + + docker exec nextcloudpi bash /usr/local/bin/ncp-diag test: needs: @@ -265,4 +267,6 @@ jobs: exit 1 } echo "CI:: Nextcloud test successful" + + docker exec nextcloudpi bash /usr/local/bin/ncp-diag -- cgit v1.2.3 From 270fc64d0c5b51d91f6d6f5bedd367cbbc1a0a0f Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 21:29:19 +0200 Subject: vm-tests.yml: Remove obsolete matrix configuration Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/release.yml | 1 - .github/workflows/vm-tests.yml | 71 ++++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 42 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69609cd5..776524b7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -144,7 +144,6 @@ jobs: board_name: OdroidC2 secrets: inherit - lxd-release: needs: - github-release diff --git a/.github/workflows/vm-tests.yml b/.github/workflows/vm-tests.yml index 89eb437b..5f2cd51a 100644 --- a/.github/workflows/vm-tests.yml +++ b/.github/workflows/vm-tests.yml @@ -239,29 +239,24 @@ jobs: image: thecalcaholic/ncp-test-automation:latest env: HCLOUD_TOKEN: "${{ secrets.TEST_AUTOMATION_HCLOUD_API_TOKEN }}" - strategy: - matrix: - test_type: [install] - include: - - test_type: install - server_address: ${{ needs.setup-installation-test-instance.outputs.server_address }} - test_result: ${{ needs.setup-installation-test-instance.result }} - test_server_id: ${{ needs.setup-installation-test-instance.outputs.test_server_id }} - version: ${{ needs.setup-installation-test-instance.outputs.version }} - fail-fast: false env: - UID: ${{ github.run_id }}-${{ matrix.test_type }} - VERSION: ${{ matrix.version }} + TEST_TYPE: install + SERVER_ADDRESS: ${{ needs.setup-installation-test-instance.outputs.server_address }} + TEST_RESULT: ${{ needs.setup-installation-test-instance.result }} + TEST_SERVER_ID: ${{ needs.setup-installation-test-instance.outputs.test_server_id }} + VERSION: ${{ needs.setup-installation-test-instance.outputs.version }} + + UID: ${{ github.run_id }}-${TEST_TYPE} steps: - name: download ssh private key from artifact store uses: actions/download-artifact@v3 - if: ${{ contains('success|failure', matrix.test_result) }} + if: ${{ contains('success|failure', env.TEST_RESULT) }} with: - name: ${{ github.run_id }}-${{ matrix.test_type }}-ssh-privkey + name: ${{ github.run_id }}-${TEST_TYPE}-ssh-privkey path: /github/workspace/.ssh - name: Shutdown server - if: ${{ contains('success|failure', matrix.test_result) }} + if: ${{ contains('success|failure', env.TEST_RESULT) }} run: | chmod 0600 /github/workspace/.ssh/automation_ssh_key export SSH_PUBLIC_KEY="$(cat /github/workspace/.ssh/automation_ssh_key.pub)" @@ -269,25 +264,25 @@ jobs: eval "$(ssh-agent)" ssh-add /github/workspace/.ssh/automation_ssh_key - ssh -o "StrictHostKeyChecking=no" -o "UserKnownHostsFile=/dev/null" "root@${{ matrix.server_address }}" < Date: Wed, 28 Sep 2022 22:02:28 +0200 Subject: run-parts.sh: Create backup before shutting down services Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/build-docker.yml | 5 +++-- build/docker/debian-ncp/run-parts.sh | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index a7c4d9bf..3d96f996 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -129,8 +129,7 @@ jobs: echo 'CONTAINER:: ==========' docker logs -f nextcloudpi |& awk '{ print "CONTAINER::" $0 }' & docker exec nextcloudpi bash -c 'tail -f /var/log/ncp.log' |& awk '{ print "NCP::" $0 }' & - cmd=(python activation_tests.py --no-gui localhost 8443 4443) - [[ "${ARCH?}" == "x86" ]] || cmd+=(--timeout 300) + [[ "${ARCH?}" == "x86" ]] || sleep 30 sleep 20 @@ -239,6 +238,7 @@ jobs: echo ":: System Tests (attempt $attempt/3) ::" python system_tests.py --no-ping --non-interactive || { echo "CI:: System test (attempt $attempt) failed!" + sleep 5 continue } success=true @@ -257,6 +257,7 @@ jobs: python nextcloud_tests.py --no-gui localhost 8443 4443 || { tail -n 20 geckodriver.log >&2 || true echo "CI:: Nextcloud test (attempt $attempt/3) failed!" + sleep 5 continue } success=true diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index 15516e7e..f87cf271 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -2,9 +2,6 @@ cleanup() { - for file in $( ls -1rv /etc/services-enabled.d ); do - /etc/services-enabled.d/"$file" stop "$1" - done if [[ -z "$NOBACKUP" ]] || [[ "$NOBACKUP" != "true" ]] then BKPDIR=/data/docker-shutdown-backups/ @@ -31,6 +28,10 @@ cleanup() echo 'WARN: Backup creation failed' fi fi + + for file in $( ls -1rv /etc/services-enabled.d ); do + /etc/services-enabled.d/"$file" stop "$1" + done exit } -- cgit v1.2.3 From 9076b3599f0dea964f41fe8e8e25535cfcaeb5be Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 22:05:54 +0200 Subject: lamp.sh: Only create systemd users if systemd is not already present Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- lamp.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lamp.sh b/lamp.sh index 00df8d14..c81c43dc 100644 --- a/lamp.sh +++ b/lamp.sh @@ -37,11 +37,13 @@ install() apache2ctl -V || true # Create systemd users to keep uids persistent between containers - addgroup --quiet --system systemd-journal - adduser --quiet -u 180 --system --group --no-create-home --home /run/systemd \ + id -u systemd-resolve || { + addgroup --quiet --system systemd-journal + adduser --quiet -u 180 --system --group --no-create-home --home /run/systemd \ --gecos "systemd Network Management" systemd-network - adduser --quiet -u 181 --system --group --no-create-home --home /run/systemd \ + adduser --quiet -u 181 --system --group --no-create-home --home /run/systemd \ --gecos "systemd Resolver" systemd-resolve + } install_with_shadow_workaround --no-install-recommends systemd $APTINSTALL -t $RELEASE php${PHPVER} php${PHPVER}-curl php${PHPVER}-gd php${PHPVER}-fpm php${PHPVER}-cli php${PHPVER}-opcache \ php${PHPVER}-mbstring php${PHPVER}-xml php${PHPVER}-zip php${PHPVER}-fileinfo php${PHPVER}-ldap \ -- cgit v1.2.3 From a6ebbb8601dd412c81915239ab7db2ea7cd754d5 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 23:43:39 +0200 Subject: 010lamp: Fail if dbdir could not be configured Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- build/docker/lamp/010lamp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/docker/lamp/010lamp b/build/docker/lamp/010lamp index cf192882..cd02a215 100755 --- a/build/docker/lamp/010lamp +++ b/build/docker/lamp/010lamp @@ -32,7 +32,7 @@ echo "Starting Apache" /usr/sbin/apache2ctl start # adjust the dbdir to the persistent storage -install_template "mysql/90-ncp.cnf.sh" "/etc/mysql/mariadb.conf.d/90-ncp.cnf" +install_template "mysql/90-ncp.cnf.sh" "/etc/mysql/mariadb.conf.d/90-ncp.cnf" || exit 1 # start echo "Starting mariaDB" -- cgit v1.2.3 From ea13af3d4d045fa225414d47fd80d2276191fcd8 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Wed, 28 Sep 2022 23:49:51 +0200 Subject: vm-tests.yml: Fix invalid use of environment variables Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- .github/workflows/vm-tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/vm-tests.yml b/.github/workflows/vm-tests.yml index 5f2cd51a..7c3024ec 100644 --- a/.github/workflows/vm-tests.yml +++ b/.github/workflows/vm-tests.yml @@ -253,7 +253,7 @@ jobs: uses: actions/download-artifact@v3 if: ${{ contains('success|failure', env.TEST_RESULT) }} with: - name: ${{ github.run_id }}-${TEST_TYPE}-ssh-privkey + name: ${{ github.run_id }}-${{ env.TEST_TYPE }}-ssh-privkey path: /github/workspace/.ssh - name: Shutdown server if: ${{ contains('success|failure', env.TEST_RESULT) }} @@ -307,7 +307,7 @@ jobs: uses: actions/download-artifact@v3 if: ${{ contains('success|failure', env.TEST_RESULT) }} with: - name: ${{ github.run_id }}-${TEST_TYPE?}-ssh-privkey + name: ${{ github.run_id }}-${{ env.TEST_TYPE }}-ssh-privkey path: /github/workspace/.ssh - name: Shutdown server if: ${{ contains('success|failure', env.TEST_RESULT) }} -- cgit v1.2.3 From 0040f89e8151684454e5341b1259b70342d7cb89 Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 29 Sep 2022 00:58:34 +0200 Subject: run-parts.sh: Simplify backup creation nextcloud-domain.sh: Only start notify_push if the nc_domain has been set Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- bin/nextcloud-domain.sh | 5 +++++ build/docker/debian-ncp/run-parts.sh | 26 +++++--------------------- build/docker/nextcloud/020nextcloud | 3 --- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/bin/nextcloud-domain.sh b/bin/nextcloud-domain.sh index 95eb0963..dfc757bb 100644 --- a/bin/nextcloud-domain.sh +++ b/bin/nextcloud-domain.sh @@ -34,3 +34,8 @@ while :; do set-nc-domain "${nc_domain}" >> /var/log/ncp.log break done + +[[ -z "$nc_domain" ]] || { + echo "Starting notify_push daemon" + start_notify_push +} diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index f87cf271..5f97ef13 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -4,29 +4,13 @@ cleanup() { if [[ -z "$NOBACKUP" ]] || [[ "$NOBACKUP" != "true" ]] then - BKPDIR=/data/docker-shutdown-backups/ + BKPDIR=/data/docker-shutdown-backups WITH_DATA=no COMPRESSED=yes - LIMIT=0 + LIMIT=5 mkdir -p "$BKPDIR" - echo "Cleanup old shutdown backups..." - skip_bkp_cleanup=0 - BKPS="$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null || skip_bkp_cleanup=1)" - if [[ "$skip_bkp_cleanup" == 0 ]] - then - while read -r bkp - do - rm -f "$BKPDIR/$bkp" - done <"$(echo "$BKPS" | tail -n +5)" - fi echo "Back up current instance..." - set +eE - if ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" - then - echo "Backup stored at '$BKPDIR/$(ls -1t "$BKPDIR"/nextcloud-bkp_*.tar.gz 2>/dev/null | head -n1)'" - else - echo 'WARN: Backup creation failed' - fi + ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" || echo 'WARN: Backup creation failed' fi for file in $( ls -1rv /etc/services-enabled.d ); do @@ -40,7 +24,7 @@ grep '/data-ro' /etc/mysql/mariadb.conf.d/90-ncp.cnf > /dev/null 2>&1 && { "to fix this now, but if you encounter any issues, please check" \ "https://github.com/nextcloud/nextcloudpi/issues/1577#issuecomment-1260830341" \ "It is likely that you will have to restore a backup" - chown -R mysql: /data/database || true + chown -R mysql:mysql /data/database || true } sed -i 's|/data-ro|/data|' "/etc/mysql/mariadb.conf.d/90-ncp.cnf" || true @@ -91,4 +75,4 @@ done echo "Init done" while true; do sleep 0.5; done -} + diff --git a/build/docker/nextcloud/020nextcloud b/build/docker/nextcloud/020nextcloud index edfe4b1d..62452c74 100755 --- a/build/docker/nextcloud/020nextcloud +++ b/build/docker/nextcloud/020nextcloud @@ -55,9 +55,6 @@ EOF echo "Provisioning" bash /usr/local/bin/ncp-provisioning.sh -echo "Starting notify_push daemon" -start_notify_push - if needs_decrypt; then echo "Waiting for user to decrypt instance" while :; do -- cgit v1.2.3 From 4d56423c6bbe23800d6db4c393183ce165a13c0f Mon Sep 17 00:00:00 2001 From: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> Date: Thu, 29 Sep 2022 02:01:36 +0200 Subject: nc-limits.sh: Limit max memory to 4GB on 32 bit systems run-parts.sh: Run backup on startup not shutdown Signed-off-by: thecalcaholic <6317548+theCalcaholic@users.noreply.github.com> --- bin/ncp/CONFIG/nc-limits.sh | 10 +++++++++- build/docker/debian-ncp/run-parts.sh | 22 ++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/bin/ncp/CONFIG/nc-limits.sh b/bin/ncp/CONFIG/nc-limits.sh index c1d0dcfc..bcbe63ea 100644 --- a/bin/ncp/CONFIG/nc-limits.sh +++ b/bin/ncp/CONFIG/nc-limits.sh @@ -9,7 +9,15 @@ # get_total_mem() { - free -b | sed -n 2p | awk '{ print $2 }' + local total_mem="$(free -b | sed -n 2p | awk '{ print $2 }')" + local MAX_32BIT=4096000000 + if [[ "$ARCH" == 'armv7' ]] && [[ $MAX_32BIT -lt "$total_mem" ]] + then + echo "$MAX_32BIT" + else + echo "$total_mem" + fi + } tmpl_innodb_buffer_pool_size() { diff --git a/build/docker/debian-ncp/run-parts.sh b/build/docker/debian-ncp/run-parts.sh index 5f97ef13..95c85fae 100755 --- a/build/docker/debian-ncp/run-parts.sh +++ b/build/docker/debian-ncp/run-parts.sh @@ -2,16 +2,6 @@ cleanup() { - if [[ -z "$NOBACKUP" ]] || [[ "$NOBACKUP" != "true" ]] - then - BKPDIR=/data/docker-shutdown-backups - WITH_DATA=no - COMPRESSED=yes - LIMIT=5 - mkdir -p "$BKPDIR" - echo "Back up current instance..." - ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" || echo 'WARN: Backup creation failed' - fi for file in $( ls -1rv /etc/services-enabled.d ); do /etc/services-enabled.d/"$file" stop "$1" @@ -73,6 +63,18 @@ done # wait for trap from 'docker stop' echo "Init done" + +if [[ -z "$NOBACKUP" ]] || [[ "$NOBACKUP" != "true" ]] +then + BKPDIR=/data/docker-startup-backups + WITH_DATA=no + COMPRESSED=yes + LIMIT=5 + mkdir -p "$BKPDIR" + echo "Back up current instance..." + ncp-backup "$BKPDIR" "$WITH_DATA" "$COMPRESSED" "$LIMIT" || echo 'WARN: Backup creation failed' +fi + while true; do sleep 0.5; done -- cgit v1.2.3