diff options
author | Mograine <ghiot.pierre@gmail.com> | 2019-11-13 11:44:48 +0300 |
---|---|---|
committer | Mograine <ghiot.pierre@gmail.com> | 2019-11-13 11:44:48 +0300 |
commit | b93628acb3d8d77775dae9778d07262c93c0aca1 (patch) | |
tree | 391d8675972878534673da0e7d9cf48106aa682e /advanced | |
parent | 193ff38ab3d6fd49a9912c5e07cd8e098267bf3a (diff) | |
parent | 29bad2fe9baed83394c4c20156d735474c1e5dba (diff) |
Merge branch 'development' of https://github.com/Mograine/pi-hole into patch-1
Diffstat (limited to 'advanced')
-rw-r--r-- | advanced/01-pihole.conf | 2 | ||||
-rw-r--r-- | advanced/Scripts/database_migration/gravity-db.sh | 44 | ||||
-rw-r--r-- | advanced/Scripts/database_migration/gravity/1_to_2.sql | 14 | ||||
-rw-r--r-- | advanced/Scripts/database_migration/gravity/2_to_3.sql | 65 | ||||
-rwxr-xr-x | advanced/Scripts/list.sh | 280 | ||||
-rwxr-xr-x | advanced/Scripts/piholeARPTable.sh | 73 | ||||
-rw-r--r-- | advanced/Scripts/piholeCheckout.sh | 3 | ||||
-rwxr-xr-x | advanced/Scripts/piholeDebug.sh | 134 | ||||
-rwxr-xr-x | advanced/Scripts/piholeLogFlush.sh | 4 | ||||
-rw-r--r-- | advanced/Scripts/query.sh | 153 | ||||
-rwxr-xr-x | advanced/Scripts/updatecheck.sh | 3 | ||||
-rwxr-xr-x | advanced/Scripts/webpage.sh | 75 | ||||
-rw-r--r-- | advanced/Templates/gravity.db.sql | 142 | ||||
-rw-r--r-- | advanced/bash-completion/pihole | 2 | ||||
-rw-r--r-- | advanced/index.php | 28 |
15 files changed, 743 insertions, 279 deletions
diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf index 5919034d..2c8b3749 100644 --- a/advanced/01-pihole.conf +++ b/advanced/01-pihole.conf @@ -18,8 +18,6 @@ # WITHIN /etc/dnsmasq.d/yourname.conf # ############################################################################### -addn-hosts=/etc/pihole/gravity.list -addn-hosts=/etc/pihole/black.list addn-hosts=/etc/pihole/local.list addn-hosts=/etc/pihole/custom.list diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh new file mode 100644 index 00000000..0fe90d8a --- /dev/null +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090 + +# Pi-hole: A black hole for Internet advertisements +# (c) 2019 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# Updates gravity.db database +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +upgrade_gravityDB(){ + local database piholeDir auditFile version + database="${1}" + piholeDir="${2}" + auditFile="${piholeDir}/auditlog.list" + + # Get database version + version="$(sqlite3 "${database}" "SELECT \"value\" FROM \"info\" WHERE \"property\" = 'version';")" + + if [[ "$version" == "1" ]]; then + # This migration script upgrades the gravity.db file by + # adding the domain_audit table + echo -e " ${INFO} Upgrading gravity database from version 1 to 2" + sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/1_to_2.sql" + version=2 + + # Store audit domains in database table + if [ -e "${auditFile}" ]; then + echo -e " ${INFO} Migrating content of ${auditFile} into new database" + # database_table_from_file is defined in gravity.sh + database_table_from_file "domain_audit" "${auditFile}" + fi + fi + if [[ "$version" == "2" ]]; then + # This migration script upgrades the gravity.db file by + # renaming the regex table to regex_blacklist, and + # creating a new regex_whitelist table + corresponding linking table and views + echo -e " ${INFO} Upgrading gravity database from version 2 to 3" + sqlite3 "${database}" < "/etc/.pihole/advanced/Scripts/database_migration/gravity/2_to_3.sql" + version=3 + fi +} diff --git a/advanced/Scripts/database_migration/gravity/1_to_2.sql b/advanced/Scripts/database_migration/gravity/1_to_2.sql new file mode 100644 index 00000000..6d57a6fe --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/1_to_2.sql @@ -0,0 +1,14 @@ +.timeout 30000 + +BEGIN TRANSACTION; + +CREATE TABLE domain_audit +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)) +); + +UPDATE info SET value = 2 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Scripts/database_migration/gravity/2_to_3.sql b/advanced/Scripts/database_migration/gravity/2_to_3.sql new file mode 100644 index 00000000..fd7c24d2 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/2_to_3.sql @@ -0,0 +1,65 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; + +ALTER TABLE regex RENAME TO regex_blacklist; + +CREATE TABLE regex_blacklist_by_group +( + regex_blacklist_id INTEGER NOT NULL REFERENCES regex_blacklist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_blacklist_id, group_id) +); + +INSERT INTO regex_blacklist_by_group SELECT * FROM regex_by_group; +DROP TABLE regex_by_group; +DROP VIEW vw_regex; +DROP TRIGGER tr_regex_update; + +CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain + FROM regex_blacklist + LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id + LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id + WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_blacklist.id; + +CREATE TRIGGER tr_regex_blacklist_update AFTER UPDATE ON regex_blacklist + BEGIN + UPDATE regex_blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + +CREATE TABLE regex_whitelist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); + +CREATE TABLE regex_whitelist_by_group +( + regex_whitelist_id INTEGER NOT NULL REFERENCES regex_whitelist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_whitelist_id, group_id) +); + +CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain + FROM regex_whitelist + LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_whitelist_id = regex_whitelist.id + LEFT JOIN "group" ON "group".id = regex_whitelist_by_group.group_id + WHERE regex_whitelist.enabled = 1 AND (regex_whitelist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex_whitelist.id; + +CREATE TRIGGER tr_regex_whitelist_update AFTER UPDATE ON regex_whitelist + BEGIN + UPDATE regex_whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + + +UPDATE info SET value = 3 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index c1d95aae..6a606665 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -11,46 +11,51 @@ # Globals basename=pihole piholeDir=/etc/"${basename}" -whitelist="${piholeDir}"/whitelist.txt -blacklist="${piholeDir}"/blacklist.txt +gravityDBfile="${piholeDir}/gravity.db" -readonly regexlist="/etc/pihole/regex.list" reload=false addmode=true verbose=true wildcard=false +web=false domList=() -listMain="" -listAlt="" +listType="" +listname="" colfile="/opt/pihole/COL_TABLE" source ${colfile} helpFunc() { - if [[ "${listMain}" == "${whitelist}" ]]; then + if [[ "${listType}" == "whitelist" ]]; then param="w" - type="white" - elif [[ "${listMain}" == "${regexlist}" && "${wildcard}" == true ]]; then + type="whitelist" + elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then param="-wild" - type="wildcard black" - elif [[ "${listMain}" == "${regexlist}" ]]; then + type="wildcard blacklist" + elif [[ "${listType}" == "regex_blacklist" ]]; then param="-regex" - type="regex black" + type="regex blacklist filter" + elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then + param="-white-wild" + type="wildcard whitelist" + elif [[ "${listType}" == "regex_whitelist" ]]; then + param="-white-regex" + type="regex whitelist filter" else param="b" - type="black" + type="blacklist" fi echo "Usage: pihole -${param} [options] <domain> <domain2 ...> Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com' -${type^}list one or more domains +${type^} one or more domains Options: - -d, --delmode Remove domain(s) from the ${type}list - -nr, --noreload Update ${type}list without refreshing dnsmasq + -d, --delmode Remove domain(s) from the ${type} + -nr, --noreload Update ${type} without reloading the DNS server -q, --quiet Make output less verbose -h, --help Show this help dialog -l, --list Display all your ${type}listed domains @@ -73,7 +78,7 @@ HandleOther() { # Check validity of domain (don't check for regex entries) if [[ "${#domain}" -le 253 ]]; then - if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then + if [[ ( "${listType}" == "regex_blacklist" || "${listType}" == "regex_whitelist" ) && "${wildcard}" == false ]]; then validDomain="${domain}" else validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check @@ -88,175 +93,152 @@ HandleOther() { fi } -PoplistFile() { - # Check whitelist file exists, and if not, create it - if [[ ! -f "${whitelist}" ]]; then - touch "${whitelist}" - fi - - # Check blacklist file exists, and if not, create it - if [[ ! -f "${blacklist}" ]]; then - touch "${blacklist}" +ProcessDomainList() { + local is_regexlist + if [[ "${listType}" == "regex_blacklist" ]]; then + # Regex black filter list + listname="regex blacklist filters" + is_regexlist=true + elif [[ "${listType}" == "regex_whitelist" ]]; then + # Regex white filter list + listname="regex whitelist filters" + is_regexlist=true + else + # Whitelist / Blacklist + listname="${listType}" + is_regexlist=false fi for dom in "${domList[@]}"; do - # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other + # Format domain into regex filter if requested + if [[ "${wildcard}" == true ]]; then + dom="(^|\\.)${dom//\./\\.}$" + fi + + # Logic: If addmode then add to desired list and remove from the other; + # if delmode then remove from desired list but do not add to the other if ${addmode}; then - AddDomain "${dom}" "${listMain}" - RemoveDomain "${dom}" "${listAlt}" + AddDomain "${dom}" "${listType}" + if ! ${is_regexlist}; then + RemoveDomain "${dom}" "${listAlt}" + fi else - RemoveDomain "${dom}" "${listMain}" + RemoveDomain "${dom}" "${listType}" fi done } AddDomain() { + local domain list num + # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input + domain="$1" list="$2" - domain=$(EscapeRegexp "$1") - - [[ "${list}" == "${whitelist}" ]] && listname="whitelist" - [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - - if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then - [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" - [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only" - bool=true - # Is the domain in the list we want to add it to? - grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false - - if [[ "${bool}" == false ]]; then - # Domain not found in the whitelist file, add it! - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${1} to ${listname}..." - fi - reload=true - # Add it to the list we want to add it to - echo "$1" >> "${list}" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" - fi - fi - elif [[ "${list}" == "${regexlist}" ]]; then - [[ -z "${type}" ]] && type="--wildcard-only" - bool=true - domain="${1}" - [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$" + # Is the domain in the list we want to add it to? + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")" - # Is the domain in the list? - # Search only for exactly matching lines - grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false + if [[ "${num}" -ne 0 ]]; then + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} already exists in ${listname}, no need to add!" + fi + return + fi - if [[ "${bool}" == false ]]; then - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} Adding ${domain} to regex list..." - fi - reload="restart" - echo "$domain" >> "${regexlist}" - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} already exists in regex list, no need to add!" - fi - fi + # Domain not found in the table, add it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Adding ${1} to the ${listname}..." fi + reload=true + # Insert only the domain here. The enabled and date_added fields will be filled + # with their default values (enabled = true, date_added = current timestamp) + sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES ('${domain}');" } RemoveDomain() { + local domain list num + # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input + domain="$1" list="$2" - domain=$(EscapeRegexp "$1") - - [[ "${list}" == "${whitelist}" ]] && listname="whitelist" - [[ "${list}" == "${blacklist}" ]] && listname="blacklist" - - if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then - bool=true - [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only" - [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only" - # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa - grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false - if [[ "${bool}" == true ]]; then - # Remove it from the other one - echo -e " ${INFO} Removing $1 from ${listname}..." - # /I flag: search case-insensitive - sed -i "/${domain}/Id" "${list}" - reload=true - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${1} does not exist in ${listname}, no need to remove!" - fi - fi - elif [[ "${list}" == "${regexlist}" ]]; then - [[ -z "${type}" ]] && type="--wildcard-only" - domain="${1}" - - [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$" - - bool=true - # Is it in the list? - grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false - if [[ "${bool}" == true ]]; then - # Remove it from the other one - echo -e " ${INFO} Removing $domain from regex list..." - local lineNumber - lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:) - sed -i "${lineNumber}d" "${list}" - reload=true - else - if [[ "${verbose}" == true ]]; then - echo -e " ${INFO} ${domain} does not exist in regex list, no need to remove!" - fi - fi + + # Is the domain in the list we want to remove it from? + num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")" + + if [[ "${num}" -eq 0 ]]; then + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} ${1} does not exist in ${list}, no need to remove!" + fi + return fi -} -# Update Gravity -Reload() { - echo "" - pihole -g --skip-download "${type:-}" + # Domain found in the table, remove it! + if [[ "${verbose}" == true ]]; then + echo -e " ${INFO} Removing ${1} from the ${listname}..." + fi + reload=true + # Remove it from the current list + sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = '${domain}';" } Displaylist() { - if [[ -f ${listMain} ]]; then - if [[ "${listMain}" == "${whitelist}" ]]; then - string="gravity resistant domains" - else - string="domains caught in the sinkhole" - fi - verbose=false - echo -e "Displaying $string:\n" + local list listname count num_pipes domain enabled status nicedate + + listname="${listType}" + data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)" + + if [[ -z $data ]]; then + echo -e "Not showing empty list" + else + echo -e "Displaying ${listname}:" count=1 - while IFS= read -r RD || [ -n "${RD}" ]; do - echo " ${count}: ${RD}" + while IFS= read -r line + do + # Count number of pipes seen in this line + # This is necessary because we can only detect the pipe separating the fields + # from the end backwards as the domain (which is the first field) may contain + # pipe symbols as they are perfectly valid regex filter control characters + num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")" + + # Extract domain and enabled status based on the obtained number of pipe characters + domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")" + enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")" + datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")" + + # Translate boolean status into human readable string + if [[ "${enabled}" -eq 1 ]]; then + status="enabled" + else + status="disabled" + fi + + # Get nice representation of numerical date stored in database + nicedate=$(date --rfc-2822 -d "@${datemod}") + + echo " ${count}: ${domain} (${status}, last modified ${nicedate})" count=$((count+1)) - done < "${listMain}" - else - echo -e " ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}" + done <<< "${data}" fi exit 0; } NukeList() { - if [[ -f "${listMain}" ]]; then - # Back up original list - cp "${listMain}" "${listMain}.bck~" - # Empty out file - echo "" > "${listMain}" - fi + sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};" } for var in "$@"; do case "${var}" in - "-w" | "whitelist" ) listMain="${whitelist}"; listAlt="${blacklist}";; - "-b" | "blacklist" ) listMain="${blacklist}"; listAlt="${whitelist}";; - "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;; - "--regex" | "regex" ) listMain="${regexlist}";; + "-w" | "whitelist" ) listType="whitelist"; listAlt="blacklist";; + "-b" | "blacklist" ) listType="blacklist"; listAlt="whitelist";; + "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;; + "--regex" | "regex" ) listType="regex_blacklist";; + "--white-regex" | "white-regex" ) listType="regex_whitelist";; + "--white-wild" | "white-wild" ) listType="regex_whitelist"; wildcard=true;; "-nr"| "--noreload" ) reload=false;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; "-h" | "--help" ) helpFunc;; "-l" | "--list" ) Displaylist;; "--nuke" ) NukeList;; + "--web" ) web=true;; * ) HandleOther "${var}";; esac done @@ -267,9 +249,13 @@ if [[ $# = 0 ]]; then helpFunc fi -PoplistFile +ProcessDomainList + +# Used on web interface +if $web; then +echo "DONE" +fi if [[ "${reload}" != false ]]; then - # Ensure that "restart" is used for Wildcard updates - Reload "${reload}" + pihole restartdns reload fi diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh new file mode 100755 index 00000000..aa45f9ad --- /dev/null +++ b/advanced/Scripts/piholeARPTable.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# shellcheck disable=SC1090 + +# Pi-hole: A black hole for Internet advertisements +# (c) 2019 Pi-hole, LLC (https://pi-hole.net) +# Network-wide ad blocking via your own hardware. +# +# ARP table interaction +# +# This file is copyright under the latest version of the EUPL. +# Please see LICENSE file for your rights under this license. + +coltable="/opt/pihole/COL_TABLE" +if [[ -f ${coltable} ]]; then + source ${coltable} +fi + +# Determine database location +# Obtain DBFILE=... setting from pihole-FTL.db +# Constructed to return nothing when +# a) the setting is not present in the config file, or +# b) the setting is commented out (e.g. "#DBFILE=...") +FTLconf="/etc/pihole/pihole-FTL.conf" +if [ -e "$FTLconf" ]; then + DBFILE="$(sed -n -e 's/^\s*DBFILE\s*=\s*//p' ${FTLconf})" +fi +# Test for empty string. Use standard path in this case. +if [ -z "$DBFILE" ]; then + DBFILE="/etc/pihole/pihole-FTL.db" +fi + + +flushARP(){ + local output + if [[ "${args[1]}" != "quiet" ]]; then + echo -ne " ${INFO} Flushing network table ..." + fi + + # Flush ARP cache to avoid re-adding of dead entries + if ! output=$(ip neigh flush all 2>&1); then + echo -e "${OVER} ${CROSS} Failed to clear ARP cache" + echo " Output: ${output}" + return 1 + fi + + # Truncate network_addresses table in pihole-FTL.db + # This needs to be done before we can truncate the network table due to + # foreign key contraints + if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network_addresses" 2>&1); then + echo -e "${OVER} ${CROSS} Failed to truncate network_addresses table" + echo " Database location: ${DBFILE}" + echo " Output: ${output}" + return 1 + fi + + # Truncate network table in pihole-FTL.db + if ! output=$(sqlite3 "${DBFILE}" "DELETE FROM network" 2>&1); then + echo -e "${OVER} ${CROSS} Failed to truncate network table" + echo " Database location: ${DBFILE}" + echo " Output: ${output}" + return 1 + fi + + if [[ "${args[1]}" != "quiet" ]]; then + echo -e "${OVER} ${TICK} Flushed network table" + fi +} + +args=("$@") + +case "${args[0]}" in + "arpflush" ) flushARP;; +esac diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index c4b07a98..673ded0b 100644 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -90,6 +90,7 @@ checkout() { local path path="development/${binary}" echo "development" > /etc/pihole/ftlbranch + chmod 644 /etc/pihole/ftlbranch elif [[ "${1}" == "master" ]] ; then # Shortcut to check out master branches echo -e " ${INFO} Shortcut \"master\" detected - checking out master branches..." @@ -104,6 +105,7 @@ checkout() { local path path="master/${binary}" echo "master" > /etc/pihole/ftlbranch + chmod 644 /etc/pihole/ftlbranch elif [[ "${1}" == "core" ]] ; then str="Fetching branches from ${piholeGitUrl}" echo -ne " ${INFO} $str" @@ -166,6 +168,7 @@ checkout() { if check_download_exists "$path"; then echo " ${TICK} Branch ${2} exists" echo "${2}" > /etc/pihole/ftlbranch + chmod 644 /etc/pihole/ftlbranch FTLinstall "${binary}" restart_service pihole-FTL enable_service pihole-FTL diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1010f26c..84e34416 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -89,16 +89,40 @@ PIHOLE_WILDCARD_CONFIG_FILE="${DNSMASQ_D_DIRECTORY}/03-wildcard.conf" WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf" #WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf" -PIHOLE_DEFAULT_AD_LISTS="${PIHOLE_DIRECTORY}/adlists.default" -PIHOLE_USER_DEFINED_AD_LISTS="${PIHOLE_DIRECTORY}/adlists.list" -PIHOLE_BLACKLIST_FILE="${PIHOLE_DIRECTORY}/blacklist.txt" -PIHOLE_BLOCKLIST_FILE="${PIHOLE_DIRECTORY}/gravity.list" PIHOLE_INSTALL_LOG_FILE="${PIHOLE_DIRECTORY}/install.log" PIHOLE_RAW_BLOCKLIST_FILES="${PIHOLE_DIRECTORY}/list.*" PIHOLE_LOCAL_HOSTS_FILE="${PIHOLE_DIRECTORY}/local.list" PIHOLE_LOGROTATE_FILE="${PIHOLE_DIRECTORY}/logrotate" PIHOLE_SETUP_VARS_FILE="${PIHOLE_DIRECTORY}/setupVars.conf" -PIHOLE_WHITELIST_FILE="${PIHOLE_DIRECTORY}/whitelist.txt" +PIHOLE_FTL_CONF_FILE="${PIHOLE_DIRECTORY}/pihole-FTL.conf" + +# Read the value of an FTL config key. The value is printed to stdout. +# +# Args: +# 1. The key to read +# 2. The default if the setting or config does not exist +get_ftl_conf_value() { + local key=$1 + local default=$2 + local value + + # Obtain key=... setting from pihole-FTL.conf + if [[ -e "$PIHOLE_FTL_CONF_FILE" ]]; then + # Constructed to return nothing when + # a) the setting is not present in the config file, or + # b) the setting is commented out (e.g. "#DBFILE=...") + value="$(sed -n -e "s/^\\s*$key=\\s*//p" ${PIHOLE_FTL_CONF_FILE})" + fi + + # Test for missing value. Use default value in this case. + if [[ -z "$value" ]]; then + value="$default" + fi + + echo "$value" +} + +PIHOLE_GRAVITY_DB_FILE="$(get_ftl_conf_value "GRAVITYDB" "${PIHOLE_DIRECTORY}/gravity.db")" PIHOLE_COMMAND="${BIN_DIRECTORY}/pihole" PIHOLE_COLTABLE_FILE="${BIN_DIRECTORY}/COL_TABLE" @@ -109,7 +133,7 @@ FTL_PORT="${RUN_DIRECTORY}/pihole-FTL.port" PIHOLE_LOG="${LOG_DIRECTORY}/pihole.log" PIHOLE_LOG_GZIPS="${LOG_DIRECTORY}/pihole.log.[0-9].*" PIHOLE_DEBUG_LOG="${LOG_DIRECTORY}/pihole_debug.log" -PIHOLE_FTL_LOG="${LOG_DIRECTORY}/pihole-FTL.log" +PIHOLE_FTL_LOG="$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/pihole-FTL.log")" PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access.log" PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error.log" @@ -142,16 +166,11 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}" "${PIHOLE_DHCP_CONFIG_FILE}" "${PIHOLE_WILDCARD_CONFIG_FILE}" "${WEB_SERVER_CONFIG_FILE}" -"${PIHOLE_DEFAULT_AD_LISTS}" -"${PIHOLE_USER_DEFINED_AD_LISTS}" -"${PIHOLE_BLACKLIST_FILE}" -"${PIHOLE_BLOCKLIST_FILE}" "${PIHOLE_INSTALL_LOG_FILE}" "${PIHOLE_RAW_BLOCKLIST_FILES}" "${PIHOLE_LOCAL_HOSTS_FILE}" "${PIHOLE_LOGROTATE_FILE}" "${PIHOLE_SETUP_VARS_FILE}" -"${PIHOLE_WHITELIST_FILE}" "${PIHOLE_COMMAND}" "${PIHOLE_COLTABLE_FILE}" "${FTL_PID}" @@ -793,7 +812,7 @@ dig_at() { # This helps emulate queries to different domains that a user might query # It will also give extra assurance that Pi-hole is correctly resolving and blocking domains local random_url - random_url=$(shuf -n 1 "${PIHOLE_BLOCKLIST_FILE}") + random_url=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity ORDER BY RANDOM() LIMIT 1") # First, do a dig on localhost to see if Pi-hole can use itself to block a domain if local_dig=$(dig +tries=1 +time=2 -"${protocol}" "${random_url}" @${local_address} +short "${record_type}"); then @@ -975,8 +994,7 @@ list_files_in_dir() { if [[ -d "${dir_to_parse}/${each_file}" ]]; then # If it's a directoy, do nothing : - elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_BLOCKLIST_FILE}" ]] || \ - [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \ + elif [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_DEBUG_LOG}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_RAW_BLOCKLIST_FILES}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_INSTALL_LOG_FILE}" ]] || \ [[ "${dir_to_parse}/${each_file}" == "${PIHOLE_SETUP_VARS_FILE}" ]] || \ @@ -1061,31 +1079,77 @@ head_tail_log() { IFS="$OLD_IFS" } -analyze_gravity_list() { - echo_current_diagnostic "Gravity list" - local head_line - local tail_line - # Put the current Internal Field Separator into another variable so it can be restored later +show_db_entries() { + local title="${1}" + local query="${2}" + local widths="${3}" + + echo_current_diagnostic "${title}" + OLD_IFS="$IFS" - # Get the lines that are in the file(s) and store them in an array for parsing later IFS=$'\r\n' + local entries=() + mapfile -t entries < <(\ + sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" \ + -cmd ".headers on" \ + -cmd ".mode column" \ + -cmd ".width ${widths}" \ + "${query}"\ + ) + + for line in "${entries[@]}"; do + log_write " ${line}" + done + + IFS="$OLD_IFS" +} + +show_groups() { + show_db_entries "Groups" "SELECT * FROM \"group\"" "4 4 30 50" +} + +show_adlists() { + show_db_entries "Adlists" "SELECT id,address,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM adlist" "4 100 7 19 19 50" + show_db_entries "Adlist groups" "SELECT * FROM adlist_by_group" "4 4" +} + +show_whitelist() { + show_db_entries "Exact whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM whitelist" "4 100 7 19 19 50" + show_db_entries "Exact whitelist groups" "SELECT * FROM whitelist_by_group" "4 4" + show_db_entries "Regex whitelist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM regex_whitelist" "4 100 7 19 19 50" + show_db_entries "Regex whitelist groups" "SELECT * FROM regex_whitelist_by_group" "4 4" +} + +show_blacklist() { + show_db_entries "Exact blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM blacklist" "4 100 7 19 19 50" + show_db_entries "Exact blacklist groups" "SELECT * FROM blacklist_by_group" "4 4" + show_db_entries "Regex blacklist" "SELECT id,domain,enabled,datetime(date_added,'unixepoch','localtime') date_added,datetime(date_modified,'unixepoch','localtime') date_modified,comment FROM regex_blacklist" "4 100 7 19 19 50" + show_db_entries "Regex blacklist groups" "SELECT * FROM regex_blacklist_by_group" "4 4" +} + +analyze_gravity_list() { + echo_current_diagnostic "Gravity List and Database" + local gravity_permissions - gravity_permissions=$(ls -ld "${PIHOLE_BLOCKLIST_FILE}") + gravity_permissions=$(ls -ld "${PIHOLE_GRAVITY_DB_FILE}") log_write "${COL_GREEN}${gravity_permissions}${COL_NC}" - local gravity_head=() - mapfile -t gravity_head < <(head -n 4 ${PIHOLE_BLOCKLIST_FILE}) - log_write " ${COL_CYAN}-----head of $(basename ${PIHOLE_BLOCKLIST_FILE})------${COL_NC}" - for head_line in "${gravity_head[@]}"; do - log_write " ${head_line}" - done + + local gravity_size + gravity_size=$(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT COUNT(*) FROM vw_gravity") + log_write " Size (excluding blacklist): ${COL_CYAN}${gravity_size}${COL_NC} entries" log_write "" - local gravity_tail=() - mapfile -t gravity_tail < <(tail -n 4 ${PIHOLE_BLOCKLIST_FILE}) - log_write " ${COL_CYAN}-----tail of $(basename ${PIHOLE_BLOCKLIST_FILE})------${COL_NC}" - for tail_line in "${gravity_tail[@]}"; do - log_write " ${tail_line}" + + OLD_IFS="$IFS" + IFS=$'\r\n' + local gravity_sample=() + mapfile -t gravity_sample < <(sqlite3 "${PIHOLE_GRAVITY_DB_FILE}" "SELECT domain FROM vw_gravity LIMIT 10") + log_write " ${COL_CYAN}----- First 10 Domains -----${COL_NC}" + + for line in "${gravity_sample[@]}"; do + log_write " ${line}" done - # Set the IFS back to what it was + + log_write "" IFS="$OLD_IFS" } @@ -1236,6 +1300,10 @@ process_status parse_setup_vars check_x_headers analyze_gravity_list +show_groups +show_adlists +show_whitelist +show_blacklist show_content_of_pihole_files parse_locale analyze_pihole_log diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 561fbce7..51e94d7c 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -39,8 +39,9 @@ if [[ "$@" == *"once"* ]]; then # Note that moving the file is not an option, as # dnsmasq would happily continue writing into the # moved file (it will have the same file handler) - cp /var/log/pihole.log /var/log/pihole.log.1 + cp -p /var/log/pihole.log /var/log/pihole.log.1 echo " " > /var/log/pihole.log + chmod 644 /var/log/pihole.log fi else # Manual flushing @@ -53,6 +54,7 @@ else echo " " > /var/log/pihole.log if [ -f /var/log/pihole.log.1 ]; then echo " " > /var/log/pihole.log.1 + chmod 644 /var/log/pihole.log.1 fi fi # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 2dead97d..1e1b159c 100644 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -11,8 +11,7 @@ # Globals piholeDir="/etc/pihole" -adListsList="$piholeDir/adlists.list" -wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf" +gravityDBfile="${piholeDir}/gravity.db" options="$*" adlist="" all="" @@ -23,27 +22,10 @@ matchType="match" colfile="/opt/pihole/COL_TABLE" source "${colfile}" -# Print each subdomain -# e.g: foo.bar.baz.com = "foo.bar.baz.com bar.baz.com baz.com com" -processWildcards() { - IFS="." read -r -a array <<< "${1}" - for (( i=${#array[@]}-1; i>=0; i-- )); do - ar="" - for (( j=${#array[@]}-1; j>${#array[@]}-i-2; j-- )); do - if [[ $j == $((${#array[@]}-1)) ]]; then - ar="${array[$j]}" - else - ar="${array[$j]}.${ar}" - fi - done - echo "${ar}" - done -} - # Scan an array of files for matching strings scanList(){ # Escape full stops - local domain="${1//./\\.}" lists="${2}" type="${3:-}" + local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" type="${3:-}" # Prevent grep from printing file path cd "$piholeDir" || exit 1 @@ -54,9 +36,14 @@ scanList(){ # /dev/null forces filename to be printed when only one list has been generated # shellcheck disable=SC2086 case "${type}" in - "exact" ) grep -i -E "(^|\\s)${domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;; - "wc" ) grep -i -o -m 1 "/${domain}/" ${lists} 2>/dev/null;; - * ) grep -i "${domain}" ${lists} /dev/null 2>/dev/null;; + "exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;; + # Create array of regexps + # Iterate through each regexp and check whether it matches the domainQuery + # If it does, print the matching regexp and continue looping + # Input 1 - regexps | Input 2 - domainQuery + "regex" ) awk 'NR==FNR{regexps[$0];next}{for (r in regexps)if($0 ~ r)print r}' \ + <(echo "${lists}") <(echo "${domain}") 2>/dev/null;; + * ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;; esac } @@ -73,11 +60,6 @@ Options: exit 0 fi -if [[ ! -e "$adListsList" ]]; then - echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}" - exit 1 -fi - # Handle valid options if [[ "${options}" == *"-bp"* ]]; then exact="exact"; blockpage=true @@ -107,49 +89,93 @@ if [[ -n "${str:-}" ]]; then exit 1 fi -# Scan Whitelist and Blacklist -lists="whitelist.txt blacklist.txt" -mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")" -if [[ -n "${results[*]}" ]]; then +scanDatabaseTable() { + local domain table type querystr result + domain="$(printf "%q" "${1}")" + table="${2}" + type="${3:-}" + + # As underscores are legitimate parts of domains, we escape them when using the LIKE operator. + # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this + # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched + # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores. + case "${type}" in + "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";; + * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";; + esac + + # Send prepared query to gravity database + result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null + if [[ -z "${result}" ]]; then + # Return early when there are no matches in this table + return + fi + + # Mark domain as having been white-/blacklist matched (global variable) wbMatch=true - # Loop through each result in order to print unique file title once + + # Print table name + if [[ -z "${blockpage}" ]]; then + echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}" + fi + + # Loop over results and print them + mapfile -t results <<< "${result}" for result in "${results[@]}"; do - fileName="${result%%.*}" if [[ -n "${blockpage}" ]]; then echo "π ${result}" exit 0 - elif [[ -n "${exact}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - else - # Only print filename title once per file - if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then - echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}" - fileName_prev="${fileName}" - fi - echo " ${result#*:}" fi + echo " ${result}" done -fi +} -# Scan Wildcards -if [[ -e "${wildcardlist}" ]]; then - # Determine all subdomains, domain and TLDs - mapfile -t wildcards <<< "$(processWildcards "${domainQuery}")" - for match in "${wildcards[@]}"; do - # Search wildcard list for matches - mapfile -t results <<< "$(scanList "${match}" "${wildcardlist}" "wc")" - if [[ -n "${results[*]}" ]]; then - if [[ -z "${wcMatch:-}" ]] && [[ -z "${blockpage}" ]]; then +scanRegexDatabaseTable() { + local domain list + domain="${1}" + list="${2}" + + # Query all regex from the corresponding database tables + mapfile -t regexList < <(sqlite3 "${gravityDBfile}" "SELECT domain FROM vw_regex_${list}" 2> /dev/null) + + # If we have regexps to process + if [[ "${#regexList[@]}" -ne 0 ]]; then + # Split regexps over a new line + str_regexList=$(printf '%s\n' "${regexList[@]}") + # Check domain against regexps + mapfile -t regexMatches < <(scanList "${domain}" "${str_regexList}" "regex") + # If there were regex matches + if [[ "${#regexMatches[@]}" -ne 0 ]]; then + # Split matching regexps over a new line + str_regexMatches=$(printf '%s\n' "${regexMatches[@]}") + # Form a "matched" message + str_message="${matchType^} found in ${COL_BOLD}Regex ${list}${COL_NC}" + # Form a "results" message + str_result="${COL_BOLD}${str_regexMatches}${COL_NC}" + # If we are displaying more than just the source of the block + if [[ -z "${blockpage}" ]]; then + # Set the wildcard match flag wcMatch=true - echo " ${matchType^} found in ${COL_BOLD}Wildcards${COL_NC}:" + # Echo the "matched" message, indented by one space + echo " ${str_message}" + # Echo the "results" message, each line indented by three spaces + # shellcheck disable=SC2001 + echo "${str_result}" | sed 's/^/ /' + else + echo "π .wildcard" + exit 0 fi - case "${blockpage}" in - true ) echo "π ${wildcardlist##*/}"; exit 0;; - * ) echo " *.${match}";; - esac fi - done -fi + fi +} + +# Scan Whitelist and Blacklist +scanDatabaseTable "${domainQuery}" "whitelist" "${exact}" +scanDatabaseTable "${domainQuery}" "blacklist" "${exact}" + +# Scan Regex table +scanRegexDatabaseTable "${domainQuery}" "whitelist" +scanRegexDatabaseTable "${domainQuery}" "blacklist" # Get version sorted *.domains filenames (without dir path) lists=("$(cd "$piholeDir" || exit 0; printf "%s\\n" -- *.domains | sort -V)") @@ -186,11 +212,8 @@ fi # Get adlist file content as array if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then - for adlistUrl in $(< "${adListsList}"); do - if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then - adlists+=("${adlistUrl}") - fi - done + # Retrieve source URLs from gravity database + mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlist;" 2> /dev/null)" fi # Print "Exact matches for" title diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index 26dc2ac2..afb03ebb 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -51,6 +51,7 @@ if [[ "$2" == "remote" ]]; then GITHUB_CORE_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/pi-hole/releases/latest' 2> /dev/null)")" echo -n "${GITHUB_CORE_VERSION}" > "${GITHUB_VERSION_FILE}" + chmod 644 "${GITHUB_VERSION_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then GITHUB_WEB_VERSION="$(json_extract tag_name "$(curl -s 'https://api.github.com/repos/pi-hole/AdminLTE/releases/latest' 2> /dev/null)")" @@ -66,6 +67,7 @@ else CORE_BRANCH="$(get_local_branch /etc/.pihole)" echo -n "${CORE_BRANCH}" > "${LOCAL_BRANCH_FILE}" + chmod 644 "${LOCAL_BRANCH_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_BRANCH="$(get_local_branch /var/www/html/admin)" @@ -79,6 +81,7 @@ else CORE_VERSION="$(get_local_version /etc/.pihole)" echo -n "${CORE_VERSION}" > "${LOCAL_VERSION_FILE}" + chmod 644 "${LOCAL_VERSION_FILE}" if [[ "${INSTALL_WEB_INTERFACE}" == true ]]; then WEB_VERSION="$(get_local_version /var/www/html/admin)" diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index dc2c83af..7568d119 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -19,6 +19,8 @@ readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf" readonly PI_HOLE_BIN_DIR="/usr/local/bin" readonly dnscustomfile="/etc/pihole/custom.list" +readonly gravityDBfile="/etc/pihole/gravity.db" + coltable="/opt/pihole/COL_TABLE" if [[ -f ${coltable} ]]; then source ${coltable} @@ -87,9 +89,9 @@ SetTemperatureUnit() { HashPassword() { # Compute password hash twice to avoid rainbow table vulnerability - return=$(echo -n ${1} | sha256sum | sed 's/\s.*$//') - return=$(echo -n ${return} | sha256sum | sed 's/\s.*$//') - echo ${return} + return=$(echo -n "${1}" | sha256sum | sed 's/\s.*$//') + return=$(echo -n "${return}" | sha256sum | sed 's/\s.*$//') + echo "${return}" } SetWebPassword() { @@ -143,18 +145,18 @@ ProcessDNSSettings() { delete_dnsmasq_setting "server" COUNTER=1 - while [[ 1 ]]; do + while true ; do var=PIHOLE_DNS_${COUNTER} if [ -z "${!var}" ]; then break; fi add_dnsmasq_setting "server" "${!var}" - let COUNTER=COUNTER+1 + (( COUNTER++ )) done # The option LOCAL_DNS_PORT is deprecated # We apply it once more, and then convert it into the current format - if [ ! -z "${LOCAL_DNS_PORT}" ]; then + if [ -n "${LOCAL_DNS_PORT}" ]; then add_dnsmasq_setting "server" "127.0.0.1#${LOCAL_DNS_PORT}" add_setting "PIHOLE_DNS_${COUNTER}" "127.0.0.1#${LOCAL_DNS_PORT}" delete_setting "LOCAL_DNS_PORT" @@ -184,7 +186,7 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 delete_dnsmasq_setting "host-record" - if [ ! -z "${HOSTRECORD}" ]; then + if [ -n "${HOSTRECORD}" ]; then add_dnsmasq_setting "host-record" "${HOSTRECORD}" fi @@ -212,6 +214,11 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423 add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_DOMAIN}/${CONDITIONAL_FORWARDING_IP}" add_dnsmasq_setting "server=/${CONDITIONAL_FORWARDING_REVERSE}/${CONDITIONAL_FORWARDING_IP}" fi + + # Prevent Firefox from automatically switching over to DNS-over-HTTPS + # This follows https://support.mozilla.org/en-US/kb/configuring-networks-disable-dns-over-https + # (sourced 7th September 2019) + add_dnsmasq_setting "server=/use-application-dns.net/" } SetDNSServers() { @@ -324,6 +331,7 @@ dhcp-option=option:router,${DHCP_ROUTER} dhcp-leasefile=/etc/pihole/dhcp.leases #quiet-dhcp " > "${dhcpconfig}" + chmod 644 "${dhcpconfig}" if [[ "${PIHOLE_DOMAIN}" != "none" ]]; then echo "domain=${PIHOLE_DOMAIN}" >> "${dhcpconfig}" @@ -395,19 +403,17 @@ SetWebUILayout() { } CustomizeAdLists() { - list="/etc/pihole/adlists.list" + local address + address="${args[3]}" if [[ "${args[2]}" == "enable" ]]; then - sed -i "\\@${args[3]}@s/^#http/http/g" "${list}" + sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'" elif [[ "${args[2]}" == "disable" ]]; then - sed -i "\\@${args[3]}@s/^http/#http/g" "${list}" + sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'" elif [[ "${args[2]}" == "add" ]]; then - if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then - echo "${args[3]}" >> ${list} - fi + sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address) VALUES ('${address}')" elif [[ "${args[2]}" == "del" ]]; then - var=$(echo "${args[3]}" | sed 's/\//\\\//g') - sed -i "/${var}/Id" "${list}" + sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'" else echo "Not permitted" return 1 @@ -519,10 +525,10 @@ Interfaces: fi if [[ "${args[2]}" == "all" ]]; then - echo -e " ${INFO} Listening on all interfaces, permiting all origins. Please use a firewall!" + echo -e " ${INFO} Listening on all interfaces, permitting all origins. Please use a firewall!" change_setting "DNSMASQ_LISTENING" "all" elif [[ "${args[2]}" == "local" ]]; then - echo -e " ${INFO} Listening on all interfaces, permiting origins from one hop away (LAN)" + echo -e " ${INFO} Listening on all interfaces, permitting origins from one hop away (LAN)" change_setting "DNSMASQ_LISTENING" "local" else echo -e " ${INFO} Listening only on interface ${PIHOLE_INTERFACE}" @@ -539,23 +545,50 @@ Interfaces: } Teleporter() { - local datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") + local datetimestamp + datetimestamp=$(date "+%Y-%m-%d_%H-%M-%S") php /var/www/html/admin/scripts/pi-hole/php/teleporter.php > "pi-hole-teleporter_${datetimestamp}.tar.gz" } +checkDomain() +{ + local domain validDomain + # Convert to lowercase + domain="${1,,}" + validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check + validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label + echo "${validDomain}" +} + addAudit() { shift # skip "-a" shift # skip "audit" - for var in "$@" + local domains validDomain + domains="" + for domain in "$@" do - echo "${var}" >> /etc/pihole/auditlog.list + # Check domain to be added. Only continue if it is valid + validDomain="$(checkDomain "${domain}")" + if [[ -n "${validDomain}" ]]; then + # Put comma in between domains when there is + # more than one domains to be added + # SQL INSERT allows adding multiple rows at once using the format + ## INSERT INTO table (domain) VALUES ('abc.de'),('fgh.ij'),('klm.no'),('pqr.st'); + if [[ -n "${domains}" ]]; then + domains="${domains}," + fi + domains="${domains}('${domain}')" + fi done + # Insert only the domain here. The date_added field will be + # filled with its default value (date_added = current timestamp) + sqlite3 "${gravityDBfile}" "INSERT INTO domain_audit (domain) VALUES ${domains};" } clearAudit() { - echo -n "" > /etc/pihole/auditlog.list + sqlite3 "${gravityDBfile}" "DELETE FROM domain_audit;" } SetPrivacyLevel() { diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql new file mode 100644 index 00000000..d0c744f4 --- /dev/null +++ b/advanced/Templates/gravity.db.sql @@ -0,0 +1,142 @@ +PRAGMA FOREIGN_KEYS=ON; + +CREATE TABLE "group" +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + enabled BOOLEAN NOT NULL DEFAULT 1, + name TEXT NOT NULL, + description TEXT +); + +CREATE TABLE whitelist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); + +CREATE TABLE whitelist_by_group +( + whitelist_id INTEGER NOT NULL REFERENCES whitelist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (whitelist_id, group_id) +); + +CREATE TABLE blacklist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); + +CREATE TABLE blacklist_by_group +( + blacklist_id INTEGER NOT NULL REFERENCES blacklist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (blacklist_id, group_id) +); + +CREATE TABLE regex +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + domain TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); + +CREATE TABLE regex_by_group +( + regex_id INTEGER NOT NULL REFERENCES regex (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (regex_id, group_id) +); + +CREATE TABLE adlist +( + id INTEGER PRIMARY KEY AUTOINCREMENT, + address TEXT UNIQUE NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 1, + date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), + comment TEXT +); + +CREATE TABLE adlist_by_group +( + adlist_id INTEGER NOT NULL REFERENCES adlist (id), + group_id INTEGER NOT NULL REFERENCES "group" (id), + PRIMARY KEY (adlist_id, group_id) +); + +CREATE TABLE gravity +( + domain TEXT PRIMARY KEY +); + +CREATE TABLE info +( + property TEXT PRIMARY KEY, + value TEXT NOT NULL +); + +INSERT INTO info VALUES("version","1"); + +CREATE VIEW vw_whitelist AS SELECT DISTINCT domain + FROM whitelist + LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id + LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id + WHERE whitelist.enabled = 1 AND (whitelist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY whitelist.id; + +CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist + BEGIN + UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + +CREATE VIEW vw_blacklist AS SELECT DISTINCT domain + FROM blacklist + LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id + LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id + WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY blacklist.id; + +CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist + BEGIN + UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + +CREATE VIEW vw_regex AS SELECT DISTINCT domain + FROM regex + LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id + LEFT JOIN "group" ON "group".id = regex_by_group.group_id + WHERE regex.enabled = 1 AND (regex_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY regex.id; + +CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex + BEGIN + UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain; + END; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address + FROM adlist + LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id + LEFT JOIN "group" ON "group".id = adlist_by_group.group_id + WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) + ORDER BY adlist.id; + +CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist + BEGIN + UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address; + END; + +CREATE VIEW vw_gravity AS SELECT domain + FROM gravity + WHERE domain NOT IN (SELECT domain from vw_whitelist); diff --git a/advanced/bash-completion/pihole b/advanced/bash-completion/pihole index 7ba0dad8..cea36060 100644 --- a/advanced/bash-completion/pihole +++ b/advanced/bash-completion/pihole @@ -7,7 +7,7 @@ _pihole() { case "${prev}" in "pihole") - opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist" + opts="admin blacklist checkout chronometer debug disable enable flush help logging query reconfigure regex restartdns status tail uninstall updateGravity updatePihole version wildcard whitelist arpflush" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) ;; "whitelist"|"blacklist"|"wildcard"|"regex") diff --git a/advanced/index.php b/advanced/index.php index 78135e1a..62e45091 100644 --- a/advanced/index.php +++ b/advanced/index.php @@ -102,20 +102,30 @@ if ($blocklistglob === array()) { die("[ERROR] There are no domain lists generated lists within <code>/etc/pihole/</code>! Please update gravity by running <code>pihole -g</code>, or repair Pi-hole using <code>pihole -r</code>."); } -// Set location of adlists file -if (is_file("/etc/pihole/adlists.list")) { - $adLists = "/etc/pihole/adlists.list"; -} elseif (is_file("/etc/pihole/adlists.default")) { - $adLists = "/etc/pihole/adlists.default"; +// Get possible non-standard location of FTL's database +$FTLsettings = parse_ini_file("/etc/pihole/pihole-FTL.conf"); +if (isset($FTLsettings["GRAVITYDB"])) { + $gravityDBFile = $FTLsettings["GRAVITYDB"]; } else { - die("[ERROR] File not found: <code>/etc/pihole/adlists.list</code>"); + $gravityDBFile = "/etc/pihole/gravity.db"; } -// Get all URLs starting with "http" or "www" from adlists and re-index array numerically -$adlistsUrls = array_values(preg_grep("/(^http)|(^www)/i", file($adLists, FILE_IGNORE_NEW_LINES))); +// Connect to gravity.db +try { + $db = new SQLite3($gravityDBFile, SQLITE3_OPEN_READONLY); +} catch (Exception $exception) { + die("[ERROR]: Failed to connect to gravity.db"); +} + +// Get all adlist addresses +$adlistResults = $db->query("SELECT address FROM vw_adlist"); +$adlistsUrls = array(); +while ($row = $adlistResults->fetchArray()) { + array_push($adlistsUrls, $row[0]); +} if (empty($adlistsUrls)) - die("[ERROR]: There are no adlist URL's found within <code>$adLists</code>"); + die("[ERROR]: There are no adlists enabled"); // Get total number of blocklists (Including Whitelist, Blacklist & Wildcard lists) $adlistsCount = count($adlistsUrls) + 3; |