From 06c07d33ee3a3154e41a3d7eca4fd778e0400267 Mon Sep 17 00:00:00 2001 From: nachoparker Date: Thu, 24 May 2018 19:28:28 +0200 Subject: Rework raspbian build system --- .dockerignore | 4 +- .gitignore | 7 +- README.md | 2 - armbian.sh | 21 +-- build-SD-odroid.sh | 11 +- build-SD.sh | 86 ---------- build-raspi.sh | 108 +++++++++++++ buildlib.sh | 267 ++++++++++++++++++++---------- docker-armhf/nextcloud/Dockerfile | 5 +- docker-armhf/nextcloudplus/Dockerfile | 15 +- docker/nextcloud/Dockerfile | 5 +- docker/nextcloudplus/Dockerfile | 15 +- etc/ncp-config.d/NFS.sh | 5 - etc/ncp-config.d/SSH.sh | 2 +- etc/ncp-config.d/UFW.sh | 6 - etc/ncp-config.d/dnsmasq.sh | 5 - etc/ncp-config.d/fail2ban.sh | 5 - etc/ncp-config.d/freeDNS.sh | 8 - etc/ncp-config.d/letsencrypt.sh | 6 - etc/ncp-config.d/modsecurity.sh | 6 - etc/ncp-config.d/nc-admin.sh | 5 - etc/ncp-config.d/nc-audit.sh | 6 - etc/ncp-config.d/nc-automount.sh | 6 - etc/ncp-config.d/nc-autoupdate-nc.sh | 5 - etc/ncp-config.d/nc-autoupdate-ncp.sh | 5 - etc/ncp-config.d/nc-backup-auto.sh | 6 - etc/ncp-config.d/nc-backup.sh | 6 - etc/ncp-config.d/nc-database.sh | 6 - etc/ncp-config.d/nc-datadir.sh | 6 - etc/ncp-config.d/nc-fix-permissions.sh | 5 - etc/ncp-config.d/nc-format-USB.sh | 5 - etc/ncp-config.d/nc-forward-ports.sh | 5 - etc/ncp-config.d/nc-httpsonly.sh | 6 - etc/ncp-config.d/nc-info.sh | 5 - etc/ncp-config.d/nc-init.sh | 6 - etc/ncp-config.d/nc-limits.sh | 6 - etc/ncp-config.d/nc-nextcloud.sh | 10 +- etc/ncp-config.d/nc-notify-updates.sh | 5 - etc/ncp-config.d/nc-passwd.sh | 5 - etc/ncp-config.d/nc-ramlogs.sh | 6 - etc/ncp-config.d/nc-restore.sh | 6 - etc/ncp-config.d/nc-rsync-auto.sh | 6 - etc/ncp-config.d/nc-rsync.sh | 6 - etc/ncp-config.d/nc-scan-auto.sh | 5 - etc/ncp-config.d/nc-scan.sh | 5 - etc/ncp-config.d/nc-snapshot-auto.sh | 6 - etc/ncp-config.d/nc-snapshot-sync.sh | 6 - etc/ncp-config.d/nc-snapshot.sh | 6 - etc/ncp-config.d/nc-static-IP.sh | 5 - etc/ncp-config.d/nc-swapfile.sh | 6 - etc/ncp-config.d/nc-update-nextcloud.sh | 6 - etc/ncp-config.d/nc-update.sh | 6 - etc/ncp-config.d/nc-webui.sh | 5 - etc/ncp-config.d/nc-wifi.sh | 6 - etc/ncp-config.d/nc-zram.sh | 6 - etc/ncp-config.d/no-ip.sh | 6 - etc/ncp-config.d/samba.sh | 5 - etc/ncp-config.d/unattended-upgrades.sh | 5 - install.sh | 8 +- installer.sh | 9 +- ncp.sh | 276 ++++++++++++++++++++++++++++++++ nextcloudplus.sh | 276 -------------------------------- post-inst.sh | 53 ++++++ prepare.sh | 73 --------- raspbian-cleanup.sh | 98 ------------ 65 files changed, 666 insertions(+), 933 deletions(-) delete mode 100755 build-SD.sh create mode 100755 build-raspi.sh create mode 100644 ncp.sh delete mode 100644 nextcloudplus.sh create mode 100644 post-inst.sh delete mode 100644 prepare.sh delete mode 100644 raspbian-cleanup.sh diff --git a/.dockerignore b/.dockerignore index 23760651..51aa7c5e 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,5 +1,7 @@ torrent/ -partial/ +tmp/ +cache/ +output/ qemu-raspbian-network/ armbian/ *.img diff --git a/.gitignore b/.gitignore index 981ff30f..0d444d35 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,10 @@ .*.swp -*.img -*.bz2 .idea/ qemu-raspbian-network/ +output/ +cache/ torrent/ armbian/ +raspbian_root +raspbian_boot +docker-armhf/raspbian_docker.img diff --git a/README.md b/README.md index 86d6fe72..d51eb7f9 100644 --- a/README.md +++ b/README.md @@ -63,8 +63,6 @@ This code also generates the [NextCloudPi docker images](https://hub.docker.com/ * Security audits with Lynis and Debsecan ( NEW 07-02-2018 ) * ZRAM ( NEW 19-03-2018 ) -Any extra can be installed independently in a running Raspbian instance through SSH. See `installer.sh` - Extras can be activated and configured using the web interface at HTTPS port 4443 diff --git a/armbian.sh b/armbian.sh index ca259fad..2efdf80b 100644 --- a/armbian.sh +++ b/armbian.sh @@ -2,7 +2,7 @@ # arguments: $RELEASE $LINUXFAMILY $BOARD $BUILD_DESKTOP -# This is the image customization script for NextCloudPlus on Armbian +# This is the image customization script for NextCloudPi on Armbian # # Copyleft 2017 by Ignacio Nunez Hernanz # GPL licensed (see end of file) * Use at your own risk! @@ -27,28 +27,9 @@ touch /.ncp-image # install NCP curl -sSL https://raw.githubusercontent.com/nextcloud/nextcloudpi/master/install.sh | bash -# permit root login in SSH -sed -i 's|^PermitRootLogin .*|PermitRootLogin yes|' /etc/ssh/sshd_config - # force change root password at first login (again) chage -d 0 root -# cleanup -apt-get autoremove -y -apt-get clean -rm /var/lib/apt/lists/* -r -rm /.ncp-image - -# cleanup all NCP options -source /usr/local/etc/library.sh -cd /usr/local/etc/ncp-config.d/ -for script in *.sh; do - cleanup_script $script -done - -# enable randomize passwords -systemctl enable nc-provisioning - # License # diff --git a/build-SD-odroid.sh b/build-SD-odroid.sh index f8358d76..5956b167 100644 --- a/build-SD-odroid.sh +++ b/build-SD-odroid.sh @@ -8,6 +8,8 @@ # Usage: ./build-SD-odroid.sh # +IMG="NextCloudPi_OdroidHC2_$( date "+%m-%d-%y" ).img" + set -e # get armbian @@ -31,13 +33,12 @@ armbian/compile.sh docker \ NO_APT_CACHER=no # pack image -IMGNAME="NextCloudPlus_OdroidHC2_$( date "+%m-%d-%y" )" -IMGFILE="$( ls -1t armbian/output/images/*.img | head -1 )" -pack_image "$IMGFILE" "$IMGNAME.img" +TAR=output/"$( basename "$IMG" .img ).tar.bz2" +pack_image "$IMG" "$TAR" # testing # TODO # uploading -create_torrent "${IMGNAME}.tar.bz2" -upload_ftp "$IMGNAME" || true +create_torrent "$TAR" +upload_ftp "$( basename "$TAR" .tar.bz2 )" diff --git a/build-SD.sh b/build-SD.sh deleted file mode 100755 index b1671e18..00000000 --- a/build-SD.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -# Batch creation of NextCloudPlus image -# -# Copyleft 2017 by Ignacio Nunez Hernanz -# GPL licensed (see end of file) * Use at your own risk! -# -# Usage: ./batch.sh -# - -set -e -source buildlib.sh # initializes $IMGNAME - -IP=$1 # First argument is the QEMU Raspbian IP address - - -[[ "$FTPPASS" == "" ]] && { - echo -e "\e[1mNo FTPPASS variable found, FTP won't work.\nYou probably want to cancel now\e[0m" - sleep 5 -} - -## BUILDING - -NC_INSTALL=etc/ncp-config.d/nc-nextcloud.sh -NC_CONFIG=etc/ncp-config.d/nc-init.sh - -IMGBASE="NextCloudPlus_RPi_$( date "+%m-%d-%y" )_base.img" - -export NO_CONFIG=1 # skip interactive configuration - -## BUILD - -download_resize_raspbian_img 1G "$IMGBASE" - -NO_HALT_STEP=1 ./installer.sh prepare.sh "$IP" "$IMGBASE" - ./installer.sh lamp.sh "$IP" "$( ls -1t *.img | head -1 )" - ./installer.sh $NC_INSTALL "$IP" "$( ls -1t *.img | head -1 )" - ./installer.sh nextcloudplus.sh "$IP" "$( ls -1t *.img | head -1 )" - ./installer.sh $NC_CONFIG "$IP" "$( ls -1t *.img | head -1 )" - ./installer.sh raspbian-cleanup.sh "$IP" "$( ls -1t *.img | head -1 )" - -## PACKING - -IMGFILE=$( ls -1t *.img | head -1 ) -IMGNAME=$( basename "$IMGFILE" _base_prepare_lamp_nc-nextcloud_nextcloudplus_nc-init_raspbian-cleanup.img ) - -[[ "$IMGNAME" != "" ]] || exit 1 - -pack_image "$IMGFILE" "$IMGNAME.img" - -## TESTING - -launch_qemu "$IMGNAME.img" & -sleep 10 -wait_SSH "$IP" -sleep 180 # Wait for the services to start. Improve this ( wait HTTP && trusted domains ) -tests/tests.py "$IP" - -ssh_pi "$IP" sudo halt - -## UPLOADING - -create_torrent "${IMGNAME}.tar.bz2" -upload_ftp "$IMGNAME" || true - -## CLEANUP - -mkdir -p partial && mv NextCloudPlus*.bz2 partial -rm -f *.img - -# License -# -# This script is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This script is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this script; if not, write to the -# Free Software Foundation, Inc., 59 Temple Place, Suite 330, -# Boston, MA 02111-1307 USA diff --git a/build-raspi.sh b/build-raspi.sh new file mode 100755 index 00000000..9fb0155a --- /dev/null +++ b/build-raspi.sh @@ -0,0 +1,108 @@ +#!/bin/bash + +# Batch creation of NextCloudPi image +# +# Copyleft 2017 by Ignacio Nunez Hernanz +# GPL licensed (see end of file) * Use at your own risk! +# +# Usage: ./batch.sh +# + +set -e +source buildlib.sh + +IP=192.168.0.145 # For QEMU automated testing +SIZE=3G # Raspbian image size +#CLEAN=1 # Pass this envvar to clean download cache +IMG="NextCloudPi_RPi_$( date "+%m-%d-%y" ).img" + +############################################################################## + +## preparations + +[[ "$FTPPASS" == "" ]] && { + echo -e "\e[1mNo FTPPASS variable found, FTP won't work.\nYou can to cancel now\e[0m" + sleep 5 +} + +[[ "$CLEAN" != "" ]] && rm -rf cache +rm -rf tmp && mkdir tmp +IMG=tmp/"$IMG" + +download_raspbian "$IMG" +resize_image "$IMG" "$SIZE" +update_boot_uuid "$IMG" # PARTUUID has changed after resize + +## BUILD NCP + +prepare_chroot_raspbian "$IMG" + +mkdir raspbian_root/tmp/ncp-build +cp etc/library.sh lamp.sh etc/ncp-config.d/{nc-nextcloud.sh,nc-init.sh} ncp.sh post-inst.sh raspbian_root/tmp/ncp-build + +PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + sudo chroot raspbian_root /bin/bash <<'EOFCHROOT' + # mark the image as an image build + touch /.ncp-image + + # update packages + apt-get update + apt-get upgrade -y + apt-get dist-upgrade -y + + # As of 03-2018, you dont get a big kernel update by doing + # this, so better be safe. Might uncomment again in the future + #$APTINSTALL rpi-update + #echo -e "y\n" | PRUNE_MODULES=1 rpi-update + + # install everything + cd /tmp/ncp-build || exit 1 + source library.sh + install_script lamp.sh + install_script nc-nextcloud.sh + activate_script nc-nextcloud.sh + install_script ncp.sh + activate_script nc-init.sh + activate_script post-inst.sh + + # harden SSH further for Raspbian + sed -i 's|^#PermitRootLogin .*|PermitRootLogin no|' /etc/ssh/sshd_config + + rm -rf /tmp/ncp-build +EOFCHROOT + +clean_chroot_raspbian + +## pack + +mkdir -p output +TAR=output/"$( basename "$IMG" .img ).tar.bz2" +pack_image "$IMG" "$TAR" + +## test + +set_static_IP "$IMG" "$IP" +test_image "$IMG" "$IP" +rm -r tmp + +# upload +create_torrent "$TAR" +upload_ftp "$( basename "$TAR" .tar.bz2 )" + + +# License +# +# This script is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This script is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this script; if not, write to the +# Free Software Foundation, Inc., 59 Temple Place, Suite 330, +# Boston, MA 02111-1307 USA diff --git a/buildlib.sh b/buildlib.sh index a7c720f8..356455b1 100644 --- a/buildlib.sh +++ b/buildlib.sh @@ -8,36 +8,27 @@ # More at ownyourbits.com # -IMGNAME=$( basename "$IMGFILE" .img )_$( basename "$INSTALL_SCRIPT" .sh ).img DBG=x -# $IMGOUT will contain the name of the last step +# $IMG is the source image +# $IP is the IP of the QEMU images +# $IMGOUT will contain the name of the generated image function launch_install_qemu() { local IMG=$1 local IP=$2 - [[ "$IP" == "" ]] && { echo "usage: launch_install_qemu