Merge pull request #2600 from pi-hole/release/v4.2v4.2

Release v4.2

11 files changed, 137 insertions, 54 deletions

diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf
index 3f4e4bc5..af46e918 100644
--- a/advanced/01-pihole.conf
+++ b/advanced/01-pihole.conf
@@ -46,3 +46,8 @@ log-facility=/var/log/pihole.log
 local-ttl=2
 
 log-async
+
+# If a DHCP client claims that its name is "wpad", ignore that.
+# This fixes a security hole. see CERT Vulnerability VU#598349
+dhcp-name-match=set:wpad-ignore,wpad
+dhcp-ignore-names=tag:wpad-ignore
diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh
index 977d1552..1bfe5e21 100644
--- a/advanced/Scripts/piholeCheckout.sh
+++ b/advanced/Scripts/piholeCheckout.sh
@@ -167,7 +167,7 @@ checkout() {
             echo "  ${TICK} Branch ${2} exists"
             echo "${2}" > /etc/pihole/ftlbranch
             FTLinstall "${binary}"
-            start_service pihole-FTL
+            restart_service pihole-FTL
             enable_service pihole-FTL
         else
             echo "  ${CROSS} Requested branch \"${2}\" is not available"
diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index cb66f97c..c840ed04 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -76,6 +76,7 @@ WEB_SERVER_CONFIG_DIRECTORY="/etc/lighttpd"
 HTML_DIRECTORY="/var/www/html"
 WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin"
 #BLOCK_PAGE_DIRECTORY="${HTML_DIRECTORY}/pihole"
+SHM_DIRECTORY="/dev/shm"
 
 # Files required by Pi-hole
 # https://discourse.pi-hole.net/t/what-files-does-pi-hole-use/1684
@@ -976,6 +977,9 @@ list_files_in_dir() {
             [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}" ]] || \
             [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_LOG_GZIPS}" ]]; then
             :
+        elif [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then
+            # SHM file - we do not want to see the content, but we want to see the files and their sizes
+            log_write "$(ls -ld "${dir_to_parse}"/"${each_file}")"
         else
             # Then, parse the file's content into an array so each line can be analyzed if need be
             for i in "${!REQUIRED_FILES[@]}"; do
@@ -1019,6 +1023,7 @@ show_content_of_pihole_files() {
     show_content_of_files_in_dir "${CRON_D_DIRECTORY}"
     show_content_of_files_in_dir "${WEB_SERVER_LOG_DIRECTORY}"
     show_content_of_files_in_dir "${LOG_DIRECTORY}"
+    show_content_of_files_in_dir "${SHM_DIRECTORY}"
 }
 
 head_tail_log() {
diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh
index 7dfa1a90..f6d4d344 100755
--- a/advanced/Scripts/version.sh
+++ b/advanced/Scripts/version.sh
@@ -136,8 +136,16 @@ errorOutput() {
 }
 
 defaultOutput() {
+    # Source the setupvars config file
+    # shellcheck disable=SC1091
+    source /etc/pihole/setupVars.conf
+
     versionOutput "pi-hole" "$@"
-    versionOutput "AdminLTE" "$@"
+
+    if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then
+        versionOutput "AdminLTE" "$@"
+    fi
+
     versionOutput "FTL" "$@"
 }
 
diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index 0b5c351e..c6852896 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -36,7 +36,7 @@ Options:
   -e, email           Set an administrative contact address for the Block Page
   -h, --help          Show this help dialog
   -i, interface       Specify dnsmasq's interface listening behavior
-  -l, privacylevel    Set privacy level (0 = lowest, 3 = highest)"
+  -l, privacylevel    Set privacy level (0 = lowest, 4 = highest)"
     exit 0
 }
 
@@ -327,6 +327,12 @@ dhcp-leasefile=/etc/pihole/dhcp.leases
         echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}"
     fi
 
+    # Sourced from setupVars
+    # shellcheck disable=SC2154
+    if [[ "${DHCP_rapid_commit}" == "true" ]]; then
+        echo "dhcp-rapid-commit" >> "${dhcpconfig}"
+    fi
+
     if [[ "${DHCP_IPv6}" == "true" ]]; then
         echo "#quiet-dhcp6
 #enable-ra
@@ -351,6 +357,7 @@ EnableDHCP() {
     change_setting "DHCP_LEASETIME" "${args[5]}"
     change_setting "PIHOLE_DOMAIN" "${args[6]}"
     change_setting "DHCP_IPv6" "${args[7]}"
+    change_setting "DHCP_rapid_commit" "${args[8]}"
 
     # Remove possible old setting from file
     delete_dnsmasq_setting "dhcp-"
diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole
index c25c570b..7ba0dad8 100644
--- a/advanced/bash-completion/pihole
+++ b/advanced/bash-completion/pihole
@@ -56,7 +56,7 @@ _pihole() {
 		;;
 		"privacylevel")
 			if ( [[ "$prev2" == "admin" ]] || [[ "$prev2" == "-a" ]] ); then
-				opts_privacy="0 1 2 3"
+				opts_privacy="0 1 2 3 4"
 				COMPREPLY=( $(compgen -W "${opts_privacy}" -- ${cur}) )
 			else 
 				return 1
diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh
index b6bf7b6d..51fb2810 100755
--- a/automated install/basic-install.sh
+++ b/automated install/basic-install.sh
@@ -115,6 +115,9 @@ else
     OVER="\\r\\033[K"
 fi
 
+# Define global binary variable
+binary="tbd"
+
 # A simple function that just echoes out our logo in ASCII format
 # This lets users know that it is a Pi-hole, LLC product
 show_ascii_berry() {
@@ -165,6 +168,20 @@ if is_command apt-get ; then
     # grep -c will return 1 retVal on 0 matches, block this throwing the set -e with an OR TRUE
     PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true"
     # Some distros vary slightly so these fixes for dependencies may apply
+    # on Ubuntu 18.04.1 LTS we need to add the universe repository to gain access to dialog and dhcpcd5
+    APT_SOURCES="/etc/apt/sources.list"
+    if awk 'BEGIN{a=1;b=0}/bionic main/{a=0}/bionic.*universe/{b=1}END{exit a + b}' ${APT_SOURCES}; then
+        if ! whiptail --defaultno --title "Dependencies Require Update to Allowed Repositories" --yesno "Would you like to enable 'universe' repository?\\n\\nThis repository is required by the following packages:\\n\\n- dhcpcd5\\n- dialog" ${r} ${c}; then
+            printf "  %b Aborting installation: dependencies could not be installed.\\n" "${CROSS}"
+            exit # exit the installer
+        else
+            printf "  %b Enabling universe package repository for Ubuntu Bionic\\n" "${INFO}"
+            cp ${APT_SOURCES} ${APT_SOURCES}.backup # Backup current repo list
+            printf "  %b Backed up current configuration to %s\\n" "${TICK}" "${APT_SOURCES}.backup"
+            add-apt-repository universe
+            printf "  %b Enabled %s\\n" "${TICK}" "'universe' repository"
+        fi
+    fi
     # Debian 7 doesn't have iproute2 so if the dry run install is successful,
     if ${PKG_MANAGER} install --dry-run iproute2 > /dev/null 2>&1; then
         # we can install it
@@ -207,7 +224,7 @@ if is_command apt-get ; then
     # These programs are stored in an array so they can be looped through later
     INSTALLER_DEPS=(apt-utils dialog debconf dhcpcd5 git ${iproute_pkg} whiptail)
     # Pi-hole itself has several dependencies that also need to be installed
-    PIHOLE_DEPS=(bc cron curl dnsutils iputils-ping lsof netcat psmisc sudo unzip wget idn2 sqlite3 libcap2-bin dns-root-data resolvconf)
+    PIHOLE_DEPS=(cron curl dnsutils iputils-ping lsof netcat psmisc sudo unzip wget idn2 sqlite3 libcap2-bin dns-root-data resolvconf libcap2)
     # The Web dashboard has some that also need to be installed
     # It's useful to separate the two since our repos are also setup as "Core" code and "Web" code
     PIHOLE_WEB_DEPS=(lighttpd ${phpVer}-common ${phpVer}-cgi ${phpVer}-${phpSqlite})
@@ -249,7 +266,7 @@ elif is_command rpm ; then
     PKG_INSTALL=(${PKG_MANAGER} install -y)
     PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l"
     INSTALLER_DEPS=(dialog git iproute newt procps-ng which)
-    PIHOLE_DEPS=(bc bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc sqlite)
+    PIHOLE_DEPS=(bind-utils cronie curl findutils nmap-ncat sudo unzip wget libidn2 psmisc sqlite libcap)
     PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo)
     LIGHTTPD_USER="lighttpd"
     LIGHTTPD_GROUP="lighttpd"
@@ -1420,9 +1437,9 @@ stop_service() {
 }
 
 # Start/Restart service passed in as argument
-start_service() {
+restart_service() {
     # Local, named variables
-    local str="Starting ${1} service"
+    local str="Restarting ${1} service"
     printf "  %b %s..." "${INFO}" "${str}"
     # If systemctl exists,
     if is_command systemctl ; then
@@ -1892,8 +1909,9 @@ installPihole() {
     installCron
     # Install the logrotate file
     installLogrotate
-    # Check if FTL is installed
-    FTLdetect || printf "  %b FTL Engine not installed\\n" "${CROSS}"
+    # Check if dnsmasq is present. If so, disable it and back up any possible
+    # config file
+    disable_dnsmasq
     # Configure the firewall
     if [[ "${useUpdateVars}" == false ]]; then
         configureFirewall
@@ -2116,7 +2134,6 @@ clone_or_update_repos() {
 # Download FTL binary to random temp directory and install FTL binary
 FTLinstall() {
     # Local, named variables
-    local binary="${1}"
     local latesttag
     local str="Downloading and Installing FTL"
     printf "  %b %s..." "${INFO}" "${str}"
@@ -2160,33 +2177,18 @@ FTLinstall() {
         # If we downloaded binary file (as opposed to text),
         if sha1sum --status --quiet -c "${binary}".sha1; then
             printf "transferred... "
-            # Stop FTL
+
+            # Stop pihole-FTL service if available
             stop_service pihole-FTL &> /dev/null
+
             # Install the new version with the correct permissions
             install -T -m 0755 "${binary}" /usr/bin/pihole-FTL
+
             # Move back into the original directory the user was in
             popd > /dev/null || { printf "Unable to return to original directory after FTL binary download.\\n"; return 1; }
-            # Install the FTL service
-            printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
-            # dnsmasq can now be stopped and disabled if it exists
-            if which dnsmasq &> /dev/null; then
-                if check_service_active "dnsmasq";then
-                    printf "  %b FTL can now resolve DNS Queries without dnsmasq running separately\\n" "${INFO}"
-                    stop_service dnsmasq
-                    disable_service dnsmasq
-                fi
-            fi
-
-            # Backup existing /etc/dnsmasq.conf if present and ensure that
-            # /etc/dnsmasq.conf contains only "conf-dir=/etc/dnsmasq.d"
-            local conffile="/etc/dnsmasq.conf"
-            if [[ -f "${conffile}" ]]; then
-                printf "  %b Backing up %s to %s.old\\n" "${INFO}" "${conffile}" "${conffile}"
-                mv "${conffile}" "${conffile}.old"
-            fi
-            # Create /etc/dnsmasq.conf
-            echo "conf-dir=/etc/dnsmasq.d" > "${conffile}"
 
+            # Installed the FTL service
+            printf "%b  %b %s\\n" "${OVER}" "${TICK}" "${str}"
             return 0
         # Otherwise,
         else
@@ -2206,6 +2208,27 @@ FTLinstall() {
     fi
 }
 
+disable_dnsmasq() {
+    # dnsmasq can now be stopped and disabled if it exists
+    if which dnsmasq &> /dev/null; then
+        if check_service_active "dnsmasq";then
+            printf "  %b FTL can now resolve DNS Queries without dnsmasq running separately\\n" "${INFO}"
+            stop_service dnsmasq
+            disable_service dnsmasq
+        fi
+    fi
+
+    # Backup existing /etc/dnsmasq.conf if present and ensure that
+    # /etc/dnsmasq.conf contains only "conf-dir=/etc/dnsmasq.d"
+    local conffile="/etc/dnsmasq.conf"
+    if [[ -f "${conffile}" ]]; then
+        printf "  %b Backing up %s to %s.old\\n" "${INFO}" "${conffile}" "${conffile}"
+        mv "${conffile}" "${conffile}.old"
+    fi
+    # Create /etc/dnsmasq.conf
+    echo "conf-dir=/etc/dnsmasq.d" > "${conffile}"
+}
+
 get_binary_name() {
     # This gives the machine architecture which may be different from the OS architecture...
     local machine
@@ -2363,7 +2386,7 @@ FTLdetect() {
     printf "\\n  %b FTL Checks...\\n\\n" "${INFO}"
 
     if FTLcheckUpdate ; then
-        FTLinstall "${binary}" || return 1
+        FTLinstall || return 1
     fi
 }
 
@@ -2446,7 +2469,7 @@ main() {
     # Start the installer
     # Verify there is enough disk space for the install
     if [[ "${skipSpaceCheck}" == true ]]; then
-        printf"  %b Skipping free disk space verification\\n" "${INFO}"
+        printf "  %b Skipping free disk space verification\\n" "${INFO}"
     else
         verifyFreeDiskSpace
     fi
@@ -2523,6 +2546,11 @@ main() {
     else
         LIGHTTPD_ENABLED=false
     fi
+    # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole
+    if ! FTLdetect; then
+        printf "  %b FTL Engine not installed\\n" "${CROSS}"
+        exit 1
+    fi
 
     # Install and log everything to a file
     installPihole | tee -a /proc/$$/fd/3
@@ -2553,7 +2581,7 @@ main() {
     if [[ "${INSTALL_WEB_SERVER}" == true ]]; then
 
         if [[ "${LIGHTTPD_ENABLED}" == true ]]; then
-            start_service lighttpd
+            restart_service lighttpd
             enable_service lighttpd
         else
             printf "  %b Lighttpd is disabled, skipping service restart\\n" "${INFO}"
@@ -2568,7 +2596,7 @@ main() {
     # Fixes a problem reported on Ubuntu 18.04 where trying to start
     # the service before enabling causes installer to exit
     enable_service pihole-FTL
-    start_service pihole-FTL
+    restart_service pihole-FTL
 
     # Download and compile the aggregated block list
     runGravity
diff --git a/manpages/pihole-FTL.conf.5 b/manpages/pihole-FTL.conf.5
index 50536279..218f5e7a 100644
--- a/manpages/pihole-FTL.conf.5
+++ b/manpages/pihole-FTL.conf.5
@@ -64,7 +64,7 @@ pihole-FTL.conf - FTL's config file
     On which port should FTL be listening?
 .br
 
-\fBPRIVACYLEVEL=0|1|2|3\fR
+\fBPRIVACYLEVEL=0|1|2|3|4\fR
 .br
     Which privacy level is used?
 .br
@@ -74,7 +74,9 @@ pihole-FTL.conf - FTL's config file
 .br
     2 - hide domains and clients
 .br
-    3 - paranoia mode (hide everything)
+    3 - anonymous mode (hide everything)
+.br
+    4 - disable all statistics
 .br
 
 \fBIGNORE_LOCALHOST=no|yes\fR
diff --git a/manpages/pihole.8 b/manpages/pihole.8
index e0845387..bd7d0933 100644
--- a/manpages/pihole.8
+++ b/manpages/pihole.8
@@ -134,7 +134,7 @@ Available commands and options:
       -i, interface     Specify dnsmasq's interface listening behavior
 .br
       -l, privacylevel  <level> Set privacy level
-                        (0 = lowest, 3 = highest)
+                        (0 = lowest, 4 = highest)
 .br
 
 \fB-c, chronometer\fR	[options]
diff --git a/pihole b/pihole
index d4681752..84a5623f 100755
--- a/pihole
+++ b/pihole
@@ -23,17 +23,6 @@ source "${colfile}"
 
 resolver="pihole-FTL"
 
-# Must be root to use this tool
-if [[ ! $EUID -eq 0 ]];then
-  if [[ -x "$(command -v sudo)" ]]; then
-    exec sudo bash "$0" "$@"
-    exit $?
-  else
-    echo -e "  ${CROSS} sudo is needed to run pihole commands.  Please run this script as root or install sudo."
-    exit 1
-  fi
-fi
-
 webpageFunc() {
   source "${PI_HOLE_SCRIPT_DIR}/webpage.sh"
   main "$@"
@@ -155,6 +144,10 @@ Time:
 
   elif [[ "${1}" == "0" ]]; then
     # Disable Pi-hole
+    if grep -cq "BLOCKING_ENABLED=false" "${setupVars}"; then
+      echo -e "  ${INFO} Blocking already disabled, nothing to do"
+      exit 0
+    fi
     if [[ -e "${gravitylist}" ]]; then
       mv "${gravitylist}" "${gravitylist}.bck"
       echo "" > "${gravitylist}"
@@ -204,6 +197,10 @@ Time:
     fi
   else
     # Enable Pi-hole
+    if grep -cq "BLOCKING_ENABLED=true" "${setupVars}"; then
+      echo -e "  ${INFO} Blocking already enabled, nothing to do"
+      exit 0
+    fi
     echo -e "  ${INFO} Enabling blocking"
     local str="Pi-hole Enabled"
 
@@ -430,6 +427,21 @@ if [[ $# = 0 ]]; then
   helpFunc
 fi
 
+case "${1}" in
+  "-h" | "help" | "--help"      ) helpFunc;;
+esac
+
+# Must be root to use this tool
+if [[ ! $EUID -eq 0 ]];then
+  if [[ -x "$(command -v sudo)" ]]; then
+    exec sudo bash "$0" "$@"
+    exit $?
+  else
+    echo -e "  ${CROSS} sudo is needed to run pihole commands.  Please run this script as root or install sudo."
+    exit 1
+  fi
+fi
+
 # Handle redirecting to specific functions based on arguments
 case "${1}" in
   "-w" | "whitelist"            ) listFunc "$@";;
diff --git a/test/test_automated_install.py b/test/test_automated_install.py
index a2593d83..853048d1 100644
--- a/test/test_automated_install.py
+++ b/test/test_automated_install.py
@@ -481,10 +481,10 @@ def test_FTL_download_aarch64_no_errors(Pihole):
     '''
     confirms only aarch64 package is downloaded for FTL engine
     '''
-    # mock uname to return generic platform
     download_binary = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    FTLinstall pihole-FTL-aarch64-linux-gnu
+    binary="pihole-FTL-aarch64-linux-gnu"
+    FTLinstall
     ''')
     expected_stdout = tick_box + ' Downloading and Installing FTL'
     assert expected_stdout in download_binary.stdout
@@ -495,10 +495,26 @@ def test_FTL_download_unknown_fails_no_errors(Pihole):
     '''
     confirms unknown binary is not downloaded for FTL engine
     '''
-    # mock uname to return generic platform
     download_binary = Pihole.run('''
     source /opt/pihole/basic-install.sh
-    FTLinstall pihole-FTL-mips
+    binary="pihole-FTL-mips"
+    FTLinstall
+    ''')
+    expected_stdout = cross_box + ' Downloading and Installing FTL'
+    assert expected_stdout in download_binary.stdout
+    error1 = 'Error: URL https://github.com/pi-hole/FTL/releases/download/'
+    assert error1 in download_binary.stdout
+    error2 = 'not found'
+    assert error2 in download_binary.stdout
+
+
+def test_FTL_download_binary_unset_no_errors(Pihole):
+    '''
+    confirms unset binary variable does not download FTL engine
+    '''
+    download_binary = Pihole.run('''
+    source /opt/pihole/basic-install.sh
+    FTLinstall
     ''')
     expected_stdout = cross_box + ' Downloading and Installing FTL'
     assert expected_stdout in download_binary.stdout