Merge pull request #3941 from pi-hole/release/v5.2.2v5.2.2

Pi-hole Core v5.2.2

7 files changed, 304 insertions, 37 deletions

diff --git a/advanced/Scripts/chronometer.sh b/advanced/Scripts/chronometer.sh
index 37fd5177..3e77f578 100755
--- a/advanced/Scripts/chronometer.sh
+++ b/advanced/Scripts/chronometer.sh
@@ -228,8 +228,14 @@ get_sys_stats() {
         mapfile -t ph_ver_raw < <(pihole -v -c 2> /dev/null | sed -n 's/^.* v/v/p')
         if [[ -n "${ph_ver_raw[0]}" ]]; then
             ph_core_ver="${ph_ver_raw[0]}"
-            ph_lte_ver="${ph_ver_raw[1]}"
-            ph_ftl_ver="${ph_ver_raw[2]}"
+            if [[ ${#ph_ver_raw[@]} -eq 2 ]]; then
+                # AdminLTE not installed
+                ph_lte_ver="(not installed)"
+                ph_ftl_ver="${ph_ver_raw[1]}"
+            else
+                ph_lte_ver="${ph_ver_raw[1]}"
+                ph_ftl_ver="${ph_ver_raw[2]}"
+            fi
         else
             ph_core_ver="-1"
         fi
diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index bc83d70a..d8d86563 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -334,7 +334,17 @@ compare_local_version_to_git_version() {
             return 1
         fi
     else
-        :
+        # There is no git directory so check if the web interface was disabled
+        local setup_vars_web_interface
+        setup_vars_web_interface=$(< ${PIHOLE_SETUP_VARS_FILE} grep ^INSTALL_WEB_INTERFACE | cut -d '=' -f2)
+        if [[ "${pihole_component}" == "Web" ]] && [[ "${setup_vars_web_interface}" == "false" ]]; then
+            log_write "${INFO} ${pihole_component}: Disabled in setupVars.conf via INSTALL_WEB_INTERFACE=false"
+        else
+            # Return an error message
+            log_write "${COL_RED}Directory ${git_dir} doesn't exist${COL_NC}"
+            # and exit with a non zero code
+            return 1
+        fi
     fi
 }
 
@@ -371,9 +381,9 @@ get_program_version() {
     echo_current_diagnostic "${program_name} version"
     # Evalutate the program we are checking, if it is any of the ones below, show the version
     case "${program_name}" in
-        "lighttpd") program_version="$(${program_name} -v |& head -n1 | cut -d '/' -f2 | cut -d ' ' -f1)"
+        "lighttpd") program_version="$(${program_name} -v 2> /dev/null | head -n1 | cut -d '/' -f2 | cut -d ' ' -f1)"
                     ;;
-        "php") program_version="$(${program_name} -v |& head -n1 | cut -d '-' -f1 | cut -d ' ' -f2)"
+        "php") program_version="$(${program_name} -v 2> /dev/null | head -n1 | cut -d '-' -f1 | cut -d ' ' -f2)"
                 ;;
         # If a match is not found, show an error
         *) echo "Unrecognized program";
diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 2aa7e8f8..26b030c8 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -73,7 +73,7 @@ CREATE TABLE domainlist_by_group
 CREATE TABLE client
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	ip TEXT NOL NULL UNIQUE,
+	ip TEXT NOT NULL UNIQUE,
 	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	comment TEXT
diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh
index 45e96354..353899c8 100755
--- a/automated install/basic-install.sh
+++ b/automated install/basic-install.sh
@@ -2451,7 +2451,7 @@ get_binary_name() {
     elif [[ "${machine}" == "x86_64" ]]; then
         # This gives the processor of packages dpkg installs (for example, "i386")
         local dpkgarch
-        dpkgarch=$(dpkg --print-processor 2> /dev/null || true)
+        dpkgarch=$(dpkg --print-processor 2> /dev/null || dpkg --print-architecture 2> /dev/null)
 
         # Special case: This is a 32 bit OS, installed on a 64 bit machine
         # -> change machine processor to download the 32 bit executable
diff --git a/gravity.sh b/gravity.sh
index 9ac38512..5a831dae 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -176,7 +176,7 @@ database_table_from_file() {
         echo "${rowid},\"${domain}\",${timestamp}" >> "${tmpFile}"
       elif [[ "${table}" == "adlist" ]]; then
         # Adlist table format
-        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
+        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"," >> "${tmpFile}"
       else
         # White-, black-, and regexlist table format
         echo "${rowid},${type},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
diff --git a/manpages/pihole-FTL.8 b/manpages/pihole-FTL.8
index 2928f2d8..c7b69d83 100644
--- a/manpages/pihole-FTL.8
+++ b/manpages/pihole-FTL.8
@@ -1,4 +1,4 @@
-.TH "Pihole-FTL" "8" "pihole-FTL" "Pi-hole" "June 2018"
+.TH "Pihole-FTL" "8" "pihole-FTL" "Pi-hole" "November 2020"
 .SH "NAME"
 pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine
 .br
@@ -10,7 +10,7 @@ pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine
 .br
 \fBpihole-FTL test\fR
 .br
-\fBpihole-FTL -v\fR
+\fBpihole-FTL -v|-vv\fR
 .br
 \fBpihole-FTL -t\fR
 .br
@@ -22,6 +22,16 @@ pihole-FTL - Pi-hole : The Faster-Than-Light (FTL) Engine
 .br
 \fBpihole-FTL dnsmasq-test\fR
 .br
+\fBpihole-FTL regex-test str\fR
+.br
+\fBpihole-FTL regex-test str rgx\fR
+.br
+\fBpihole-FTL lua\fR
+.br
+\fBpihole-FTL luac\fR
+.br
+\fBpihole-FTL dhcp-discover\fR
+.br
 \fBpihole-FTL --\fR (\fBoptions\fR)
 .br
 
@@ -65,6 +75,11 @@ Command line arguments
     Don't start FTL, show only version
 .br
 
+\fB-vv\fR
+.br
+    Don't start FTL, show verbose version information of embedded applications
+.br
+
 \fB-t, tag\fR
 .br
     Don't start FTL, show only git tag
@@ -90,6 +105,31 @@ Command line arguments
     Test resolver config file syntax
 .br
 
+\fBregex-test str\fR
+.br
+    Test str against all regular expressions in the database
+.br
+
+\fBregex-test str rgx\fR
+.br
+    Test str against regular expression given by rgx
+.br
+
+\fBlua\fR
+.br
+    Start the embedded Lua interpreter
+.br
+
+\fBluac\fR
+.br
+    Execute the embedded Lua compiler
+.br
+
+\fBdhcp-discover\fR
+.br
+    Discover DHCP servers in the local network
+.br
+
 \fB--\fR  (options)
 .br
     Pass options to internal dnsmasq resolver
diff --git a/manpages/pihole-FTL.conf.5 b/manpages/pihole-FTL.conf.5
index 5cb0481e..42405436 100644
--- a/manpages/pihole-FTL.conf.5
+++ b/manpages/pihole-FTL.conf.5
@@ -1,4 +1,4 @@
-.TH "pihole-FTL.conf" "5" "pihole-FTL.conf" "pihole-FTL.conf" "June 2018"
+.TH "pihole-FTL.conf" "5" "pihole-FTL.conf" "pihole-FTL.conf" "November 2020"
 .SH "NAME"
 
 pihole-FTL.conf - FTL's config file
@@ -7,43 +7,154 @@ pihole-FTL.conf - FTL's config file
 
 /etc/pihole/pihole-FTL.conf will be read by \fBpihole-FTL(8)\fR on startup.
 .br
+For each setting the option shown first is the default.
+.br
 
-\fBSOCKET_LISTENING=localonly|all\fR
+\fBBLOCKINGMODE=IP|IP-AAAA-NODATA|NODATA|NXDOMAIN|NULL\fR
+.br
+    How should FTL reply to blocked queries?
+
+    IP             - Pi-hole's IPs for blocked domains
+
+    IP-AAAA-NODATA - Pi-hole's IP + NODATA-IPv6 for blocked domains
+
+    NODATA         - Using NODATA for blocked domains
+
+    NXDOMAIN       - NXDOMAIN for blocked domains
+
+    NULL           - Null IPs for blocked domains
+.br
+
+\fBCNAME_DEEP_INSPECT=true|false\fR
+.br
+    Use this option to disable deep CNAME inspection. This might be beneficial for very low-end devices.
+.br
+
+\fBBLOCK_ESNI=true|false\fR
+.br
+    Block requests to _esni.* sub-domains.
+.br
+
+\fBMAXLOGAGE=24.0\fR
+.br
+    Up to how many hours of queries should be imported from the database and logs?
 .br
-    Listen only for local socket connections or permit all connections
+    Maximum is 744 (31 days)
 .br
 
-\fBQUERY_DISPLAY=yes|no\fR
+\fBPRIVACYLEVEL=0|1|2|3|4\fR
+.br
+    Privacy level used to collect Pi-hole statistics.
+.br
+    0 - show everything
 .br
-    Display all queries? Set to no to hide query display
+    1 - hide domains
+.br
+    2 - hide domains and clients
+.br
+    3 - anonymous mode (hide everything)
+.br
+    4 - disable all statistics
+.br
+
+\fBIGNORE_LOCALHOST=no|yes\fR
+.br
+    Should FTL ignore queries coming from the local machine?
 .br
 
 \fBAAAA_QUERY_ANALYSIS=yes|no\fR
 .br
-    Allow FTL to analyze AAAA queries from pihole.log?
+    Should FTL analyze AAAA queries?
+.br
+
+\fBANALYZE_ONLY_A_AND_AAAA=false|true\fR
+.br
+    Should FTL only analyze A and AAAA queries?
+.br
+
+\fBSOCKET_LISTENING=localonly|all\fR
+.br
+    Listen only for local socket connections on the API port or permit all connections.
+.br
+
+\fBFTLPORT=4711\fR
+.br
+    On which port should FTL be listening?
 .br
 
 \fBRESOLVE_IPV6=yes|no\fR
 .br
-    Should FTL try to resolve IPv6 addresses to host names?
+    Should FTL try to resolve IPv6 addresses to hostnames?
 .br
 
 \fBRESOLVE_IPV4=yes|no\fR
 .br
-    Should FTL try to resolve IPv4 addresses to host names?
+    Should FTL try to resolve IPv4 addresses to hostnames?
 .br
 
-\fBMAXDBDAYS=365\fR
+\fBDELAY_STARTUP=0\fR
+.br
+    Time in seconds (between 0 and 300) to delay FTL startup.
+.br
+
+\fBNICE=-10\fR
+.br
+    Set the niceness of the Pi-hole FTL process.
+.br
+    Can be disabled altogether by setting a value of -999.
+.br
+
+\fBNAMES_FROM_NETDB=true|false\fR
+.br
+    Control whether FTL should use a fallback option and try to obtain client names from checking the network table.
+.br
+    E.g. IPv6 clients without a hostname will be compared via MAC address to known clients.
+.br
+
+\fB\fBREFRESH_HOSTNAMES=IPV4|ALL|NONE\fR
+.br
+    Change how (and if) hourly PTR requests are made to check for changes in client and upstream server hostnames:
+.br
+    IPV4 - Do the hourly PTR lookups only for IPv4 addresses resolving issues in networks with many short-lived PE IPv6 addresses.
+.br
+    ALL  - Do the hourly PTR lookups for all addresses. This can create a lot of PTR queries in networks with many IPv6 addresses.
+.br
+    NONE - Don't do hourly PTR lookups. Look up hostnames once (when first seeing a client) and never again. Future hostname changes may be missed.
+.br
+
+\fBMAXNETAGE=365\fR
+.br
+    IP addresses (and associated host names) older than the specified number of days are removed.
+.br
+    This avoids dead entries in the network overview table.
+.br
+
+\fBEDNS0_ECS=true|false\fR
+.br
+    Should we overwrite the query source when client information is provided through EDNS0 client subnet (ECS) information?
+.br
+
+\fBPARSE_ARP_CACHE=true|false\fR
+.br
+    Parse ARP cache to fill network overview table.
+.br
+
+\fBDBIMPORT=yes|no\fR
 .br
-    How long should queries be stored in the database?
+    Should FTL load information from the database on startup to be aware of the most recent history?
 .br
-    Setting this to 0 disables the database
+
+\fBMAXDBDAYS=365\fR
+.br
+    How long should queries be stored in the database? Setting this to 0 disables the database
 .br
 
 \fBDBINTERVAL=1.0\fR
 .br
     How often do we store queries in FTL's database [minutes]?
 .br
+    Accepts value between 0.1 (6 sec) and 1440 (1 day)
+.br
 
 \fBDBFILE=/etc/pihole/pihole-FTL.db\fR
 .br
@@ -52,43 +163,143 @@ pihole-FTL.conf - FTL's config file
     Setting this to DBFILE= disables the database altogether
 .br
 
-\fBMAXLOGAGE=24.0\fR
+\fBLOGFILE=/var/log/pihole-FTL.log\fR
 .br
-    Up to how many hours of queries should be imported from the database and logs?
+    The location of FTL's log file.
 .br
-    Maximum is 744 (31 days)
+
+\fBPIDFILE=/run/pihole-FTL.pid\fR
+.br
+    The file which contains the PID of FTL's main process.
 .br
 
-\fBFTLPORT=4711\fR
+\fBPORTFILE=/run/pihole-FTL.port\fR
 .br
-    On which port should FTL be listening?
+    Specify path and filename where the FTL process will write its API port number.
 .br
 
-\fBPRIVACYLEVEL=0|1|2|3\fR
+\fBSOCKETFILE=/run/pihole/FTL.sock\fR
 .br
-    Which privacy level is used?
+    The file containing the socket FTL's API is listening on.
 .br
-    0 - show everything
+
+\fBSETUPVARSFILE=/etc/pihole/setupVars.conf\fR
 .br
-    1 - hide domains
+    The config file of Pi-hole containing, e.g., the current blocking status (do not change).
 .br
-    2 - hide domains and clients
+
+\fBMACVENDORDB=/etc/pihole/macvendor.db\fR
 .br
-    3 - anonymous mode (hide everything)
+    The database containing MAC -> Vendor information for the network table.
 .br
 
-\fBIGNORE_LOCALHOST=no|yes\fR
+\fBGRAVITYDB=/etc/pihole/gravity.db\fR
 .br
-    Should FTL ignore queries coming from the local machine?
+    Specify path and filename of FTL's SQLite3 gravity database. This database contains all domains relevant for Pi-hole's DNS blocking.
 .br
 
-\fBBLOCKINGMODE=IP|IP-AAAA-NODATA|NXDOMAIN|NULL\fR
+\fBDEBUG_ALL=false|true\fR
 .br
-    How should FTL reply to blocked queries?
+    Enable all debug flags. If this is set to true, all other debug config options are ignored.
+.br
+
+\fBDEBUG_DATABASE=false|true\fR
+.br
+    Print debugging information about database actions such as SQL statements and performance.
+.br
+
+\fBDEBUG_NETWORKING=false|true\fR
+.br
+    Prints a list of the detected network interfaces on the startup of FTL.
 .br
 
-For each setting, the option shown first is the default.
+\fBDEBUG_LOCKS=false|true\fR
+.br
+    Print information about shared memory locks.
+.br
+    Messages will be generated when waiting, obtaining, and releasing a lock.
+.br
+
+\fBDEBUG_QUERIES=false|true\fR
+.br
+    Print extensive DNS query information (domains, types, replies, etc.).
+.br
+
+\fBDEBUG_FLAGS=false|true\fR
+.br
+    Print flags of queries received by the DNS hooks.
+.br
+    Only effective when \fBDEBUG_QUERIES\fR is enabled as well.
+
+\fBDEBUG_SHMEM=false|true\fR
+.br
+    Print information about shared memory buffers.
+.br
+    Messages are either about creating or enlarging shmem objects or string injections.
 .br
+
+\fBDEBUG_GC=false|true\fR
+.br
+    Print information about garbage collection (GC):
+.br
+    What is to be removed, how many have been removed and how long did GC take.
+.br
+
+\fBDEBUG_ARP=false|true\fR
+.br
+    Print information about ARP table processing:
+.br
+    How long did parsing take, whether read MAC addresses are valid, and if the macvendor.db file exists.
+.br
+
+\fBDEBUG_REGEX=false|true\fR
+.br
+    Controls if FTL should print extended details about regex matching.
+.br
+
+\fBDEBUG_API=false|true\fR
+.br
+    Print extra debugging information during telnet API calls.
+.br
+    Currently only used to send extra information when getting all queries.
+.br
+
+\fBDEBUG_OVERTIME=false|true\fR
+.br
+    Print information about overTime memory operations, such as initializing or moving overTime slots.
+.br
+
+\fBDEBUG_EXTBLOCKED=false|true\fR
+.br
+    Print information about why FTL decided that certain queries were recognized as being externally blocked.
+.br
+
+\fBDEBUG_CAPS=false|true\fR
+.br
+    Print information about POSIX capabilities granted to the FTL process.
+.br
+    The current capabilities are printed on receipt of SIGHUP i.e. after executing `killall -HUP pihole-FTL`.
+.br
+
+\fBDEBUG_DNSMASQ_LINES=false|true\fR
+.br
+    Print file and line causing a dnsmasq event into FTL's log files.
+.br
+    This is handy to implement additional hooks missing from FTL.
+.br
+
+\fBDEBUG_VECTORS=false|true\fR
+.br
+    FTL uses dynamically allocated vectors for various tasks.
+.br
+    This config option enables extensive debugging information such as information about allocation, referencing, deletion, and appending.
+.br
+
+\fBDEBUG_RESOLVER=false|true\fR
+.br
+    Extensive information about hostname resolution like which DNS servers are used in the first and second hostname resolving tries.
+.br
+
 .SH "SEE ALSO"
 
 \fBpihole\fR(8), \fBpihole-FTL\fR(8)