Merge branch 'development' into new/internal-blocking

Signed-off-by: DL6ER <dl6er@dl6er.de>

11 files changed, 262 insertions, 155 deletions

diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf
index cd74e186..2c8b3749 100644
--- a/advanced/01-pihole.conf
+++ b/advanced/01-pihole.conf
@@ -19,6 +19,7 @@
 ###############################################################################
 
 addn-hosts=/etc/pihole/local.list
+addn-hosts=/etc/pihole/custom.list
 
 domain-needed
 
diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh
index 1fe6a61f..508d9c56 100644
--- a/advanced/Scripts/database_migration/gravity-db.sh
+++ b/advanced/Scripts/database_migration/gravity-db.sh
@@ -44,17 +44,24 @@ upgrade_gravityDB(){
 		version=3
 	fi
 	if [[ "$version" == "3" ]]; then
-		# This migration script upgrades the gravity and list views
-		# implementing necessary changes for per-client blocking
+		# This migration script unifies the formally separated domain
+		# lists into a single table with a UNIQUE domain constraint
 		echo -e "  ${INFO} Upgrading gravity database from version 3 to 4"
 		sqlite3 "${database}" < "${scriptPath}/3_to_4.sql"
 		version=4
 	fi
 	if [[ "$version" == "4" ]]; then
-		# This migration script upgrades the adlist view
-		# to return an ID used in gravity.sh
+		# This migration script upgrades the gravity and list views
+		# implementing necessary changes for per-client blocking
 		echo -e "  ${INFO} Upgrading gravity database from version 4 to 5"
 		sqlite3 "${database}" < "${scriptPath}/4_to_5.sql"
 		version=5
 	fi
+	if [[ "$version" == "5" ]]; then
+		# This migration script upgrades the adlist view
+		# to return an ID used in gravity.sh
+		echo -e "  ${INFO} Upgrading gravity database from version 5 to 6"
+		sqlite3 "${database}" < "${scriptPath}/5_to_6.sql"
+		version=6
+	fi
 }
diff --git a/advanced/Scripts/database_migration/gravity/3_to_4.sql b/advanced/Scripts/database_migration/gravity/3_to_4.sql
index 182d24a1..352b1baa 100644
--- a/advanced/Scripts/database_migration/gravity/3_to_4.sql
+++ b/advanced/Scripts/database_migration/gravity/3_to_4.sql
@@ -4,67 +4,93 @@ PRAGMA FOREIGN_KEYS=OFF;
 
 BEGIN TRANSACTION;
 
-DROP TABLE gravity;
-CREATE TABLE gravity
+CREATE TABLE domainlist
 (
-	domain TEXT NOT NULL,
-	adlist_id INTEGER NOT NULL REFERENCES adlist (id),
-	PRIMARY KEY(domain, adlist_id)
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	type INTEGER NOT NULL DEFAULT 0,
+	domain TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+
+ALTER TABLE whitelist ADD COLUMN type INTEGER;
+UPDATE whitelist SET type = 0;
+INSERT INTO domainlist (type,domain,enabled,date_added,date_modified,comment)
+    SELECT type,domain,enabled,date_added,date_modified,comment FROM whitelist;
+
+ALTER TABLE blacklist ADD COLUMN type INTEGER;
+UPDATE blacklist SET type = 1;
+INSERT INTO domainlist (type,domain,enabled,date_added,date_modified,comment)
+    SELECT type,domain,enabled,date_added,date_modified,comment FROM blacklist;
+
+ALTER TABLE regex_whitelist ADD COLUMN type INTEGER;
+UPDATE regex_whitelist SET type = 2;
+INSERT INTO domainlist (type,domain,enabled,date_added,date_modified,comment)
+    SELECT type,domain,enabled,date_added,date_modified,comment FROM regex_whitelist;
+
+ALTER TABLE regex_blacklist ADD COLUMN type INTEGER;
+UPDATE regex_blacklist SET type = 3;
+INSERT INTO domainlist (type,domain,enabled,date_added,date_modified,comment)
+    SELECT type,domain,enabled,date_added,date_modified,comment FROM regex_blacklist;
+
+DROP TABLE whitelist_by_group;
+DROP TABLE blacklist_by_group;
+DROP TABLE regex_whitelist_by_group;
+DROP TABLE regex_blacklist_by_group;
+CREATE TABLE domainlist_by_group
+(
+	domainlist_id INTEGER NOT NULL REFERENCES domainlist (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (domainlist_id, group_id)
 );
 
-DROP VIEW vw_gravity;
-CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id
-    FROM gravity
-    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id
-    LEFT JOIN adlist ON adlist.id = gravity.adlist_id
-    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
-    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1);
+DROP TRIGGER tr_whitelist_update;
+DROP TRIGGER tr_blacklist_update;
+DROP TRIGGER tr_regex_whitelist_update;
+DROP TRIGGER tr_regex_blacklist_update;
+CREATE TRIGGER tr_domainlist_update AFTER UPDATE ON domainlist
+    BEGIN
+      UPDATE domainlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
 
 DROP VIEW vw_whitelist;
-CREATE VIEW vw_whitelist AS SELECT domain, whitelist.id AS id, whitelist_by_group.group_id AS group_id
-    FROM whitelist
-    LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id
-    LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id
-    WHERE whitelist.enabled = 1 AND (whitelist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY whitelist.id;
+CREATE VIEW vw_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 0
+    ORDER BY domainlist.id;
 
 DROP VIEW vw_blacklist;
-CREATE VIEW vw_blacklist AS SELECT domain, blacklist.id AS id, blacklist_by_group.group_id AS group_id
-    FROM blacklist
-    LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id
-    LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id
-    WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY blacklist.id;
+CREATE VIEW vw_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 1
+    ORDER BY domainlist.id;
 
 DROP VIEW vw_regex_whitelist;
-CREATE VIEW vw_regex_whitelist AS SELECT DISTINCT domain, regex_whitelist.id AS id, regex_whitelist_by_group.group_id AS group_id
-    FROM regex_whitelist
-    LEFT JOIN regex_whitelist_by_group ON regex_whitelist_by_group.regex_whitelist_id = regex_whitelist.id
-    LEFT JOIN "group" ON "group".id = regex_whitelist_by_group.group_id
-    WHERE regex_whitelist.enabled = 1 AND (regex_whitelist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY regex_whitelist.id;
+CREATE VIEW vw_regex_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 2
+    ORDER BY domainlist.id;
 
 DROP VIEW vw_regex_blacklist;
-CREATE VIEW vw_regex_blacklist AS SELECT DISTINCT domain, regex_blacklist.id AS id, regex_blacklist_by_group.group_id AS group_id
-    FROM regex_blacklist
-    LEFT JOIN regex_blacklist_by_group ON regex_blacklist_by_group.regex_blacklist_id = regex_blacklist.id
-    LEFT JOIN "group" ON "group".id = regex_blacklist_by_group.group_id
-    WHERE regex_blacklist.enabled = 1 AND (regex_blacklist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY regex_blacklist.id;
-
-CREATE TABLE client
-(
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	ip TEXT NOL NULL UNIQUE
-);
-
-CREATE TABLE client_by_group
-(
-	client_id INTEGER NOT NULL REFERENCES client (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (client_id, group_id)
-);
+CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 3
+    ORDER BY domainlist.id;
 
 UPDATE info SET value = 4 WHERE property = 'version';
 
-COMMIT;
+COMMIT;
\ No newline at end of file
diff --git a/advanced/Scripts/database_migration/gravity/4_to_5.sql b/advanced/Scripts/database_migration/gravity/4_to_5.sql
index 22b75d58..2ad906fc 100644
--- a/advanced/Scripts/database_migration/gravity/4_to_5.sql
+++ b/advanced/Scripts/database_migration/gravity/4_to_5.sql
@@ -4,14 +4,35 @@ PRAGMA FOREIGN_KEYS=OFF;
 
 BEGIN TRANSACTION;
 
-DROP VIEW vw_adlist;
-CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id
-    FROM adlist
-    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id
+DROP TABLE gravity;
+CREATE TABLE gravity
+(
+	domain TEXT NOT NULL,
+	adlist_id INTEGER NOT NULL REFERENCES adlist (id),
+	PRIMARY KEY(domain, adlist_id)
+);
+
+DROP VIEW vw_gravity;
+CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id
+    FROM gravity
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id
+    LEFT JOIN adlist ON adlist.id = gravity.adlist_id
     LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
-    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY adlist.id;
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1);
+
+CREATE TABLE client
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	ip TEXT NOL NULL UNIQUE
+);
+
+CREATE TABLE client_by_group
+(
+	client_id INTEGER NOT NULL REFERENCES client (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (client_id, group_id)
+);
 
 UPDATE info SET value = 5 WHERE property = 'version';
 
-COMMIT;
+COMMIT;
\ No newline at end of file
diff --git a/advanced/Scripts/database_migration/gravity/5_to_6.sql b/advanced/Scripts/database_migration/gravity/5_to_6.sql
new file mode 100644
index 00000000..d2bb3145
--- /dev/null
+++ b/advanced/Scripts/database_migration/gravity/5_to_6.sql
@@ -0,0 +1,18 @@
+.timeout 30000
+
+PRAGMA FOREIGN_KEYS=OFF;
+
+BEGIN TRANSACTION;
+
+DROP VIEW vw_adlist;
+CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id
+    FROM adlist
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY adlist.id;
+
+UPDATE info SET value = 6 WHERE property = 'version';
+
+COMMIT;
+
diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 6a606665..c5bf5b2a 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -21,68 +21,78 @@ web=false
 
 domList=()
 
-listType=""
-listname=""
+typeId=""
 
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
 
+# IDs are hard-wired to domain interpretation in the gravity database scheme
+# Clients (including FTL) will read them through the corresponding views
+readonly whitelist="0"
+readonly blacklist="1"
+readonly regex_whitelist="2"
+readonly regex_blacklist="3"
+
+GetListnameFromTypeId() {
+    if [[ "$1" == "${whitelist}" ]]; then
+        echo "whitelist"
+    elif  [[ "$1" == "${blacklist}" ]]; then
+        echo "blacklist"
+    elif  [[ "$1" == "${regex_whitelist}" ]]; then
+        echo "regex whitelist"
+    elif  [[ "$1" == "${regex_blacklist}" ]]; then
+        echo "regex blacklist"
+    fi
+}
 
-helpFunc() {
-    if [[ "${listType}" == "whitelist" ]]; then
-        param="w"
-        type="whitelist"
-    elif [[ "${listType}" == "regex_blacklist" && "${wildcard}" == true ]]; then
-        param="-wild"
-        type="wildcard blacklist"
-    elif [[ "${listType}" == "regex_blacklist" ]]; then
-        param="-regex"
-        type="regex blacklist filter"
-    elif [[ "${listType}" == "regex_whitelist" && "${wildcard}" == true ]]; then
-        param="-white-wild"
-        type="wildcard whitelist"
-    elif [[ "${listType}" == "regex_whitelist" ]]; then
-        param="-white-regex"
-        type="regex whitelist filter"
-    else
-        param="b"
-        type="blacklist"
+GetListParamFromTypeId() {
+    if [[ "${typeId}" == "${whitelist}" ]]; then
+        echo "w"
+    elif  [[ "${typeId}" == "${blacklist}" ]]; then
+        echo "b"
+    elif  [[ "${typeId}" == "${regex_whitelist}" && "${wildcard}" == true ]]; then
+        echo "-white-wild"
+    elif  [[ "${typeId}" == "${regex_whitelist}" ]]; then
+        echo "-white-regex"
+    elif  [[ "${typeId}" == "${regex_blacklist}" && "${wildcard}" == true ]]; then
+        echo "-wild"
+    elif  [[ "${typeId}" == "${regex_blacklist}" ]]; then
+        echo "-regex"
     fi
+}
+
+helpFunc() {
+    local listname param
+
+    listname="$(GetListnameFromTypeId "${typeId}")"
+    param="$(GetListParamFromTypeId)"
 
     echo "Usage: pihole -${param} [options] <domain> <domain2 ...>
 Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com'
-${type^} one or more domains
+${listname^} one or more domains
 
 Options:
-  -d, --delmode       Remove domain(s) from the ${type}
-  -nr, --noreload     Update ${type} without reloading the DNS server
+  -d, --delmode       Remove domain(s) from the ${listname}
+  -nr, --noreload     Update ${listname} without reloading the DNS server
   -q, --quiet         Make output less verbose
   -h, --help          Show this help dialog
-  -l, --list          Display all your ${type}listed domains
+  -l, --list          Display all your ${listname}listed domains
   --nuke              Removes all entries in a list"
 
   exit 0
 }
 
-EscapeRegexp() {
-    # This way we may safely insert an arbitrary
-    # string in our regular expressions
-    # This sed is intentionally executed in three steps to ease maintainability
-    # The first sed removes any amount of leading dots
-    echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g"
-}
-
-HandleOther() {
+ValidateDomain() {
     # Convert to lowercase
     domain="${1,,}"
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ ( "${listType}" == "regex_blacklist" || "${listType}" == "regex_whitelist" ) && "${wildcard}" == false ]]; then
+        if [[ ( "${typeId}" == "${regex_blacklist}" || "${typeId}" == "${regex_whitelist}" ) && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
-            validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
-            validDomain=$(grep -P "^[^\\.]{1,63}(\\.[^\\.]{1,63})*$" <<< "${validDomain}") # Length of each label
+            # Use regex to check the validity of the passed domain. see https://regexr.com/3abjr
+            validDomain=$(grep -P "^((?!-))(xn--)?[a-z0-9][a-z0-9-_]{0,61}[a-z0-9]{0,1}\.(xn--)?([a-z0-9\-]{1,61}|[a-z0-9-]{1,30}\.[a-z]{2,})$" <<< "${domain}")
         fi
     fi
 
@@ -94,21 +104,6 @@ HandleOther() {
 }
 
 ProcessDomainList() {
-    local is_regexlist
-    if [[ "${listType}" == "regex_blacklist" ]]; then
-        # Regex black filter list
-        listname="regex blacklist filters"
-        is_regexlist=true
-    elif [[ "${listType}" == "regex_whitelist" ]]; then
-        # Regex white filter list
-        listname="regex whitelist filters"
-        is_regexlist=true
-    else
-        # Whitelist / Blacklist
-        listname="${listType}"
-        is_regexlist=false
-    fi
-
     for dom in "${domList[@]}"; do
         # Format domain into regex filter if requested
         if [[ "${wildcard}" == true ]]; then
@@ -118,77 +113,82 @@ ProcessDomainList() {
         # Logic: If addmode then add to desired list and remove from the other;
         # if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
-            AddDomain "${dom}" "${listType}"
-            if ! ${is_regexlist}; then
-                RemoveDomain "${dom}" "${listAlt}"
-            fi
+            AddDomain "${dom}"
         else
-            RemoveDomain "${dom}" "${listType}"
+            RemoveDomain "${dom}"
         fi
   done
 }
 
 AddDomain() {
-    local domain list num
-    # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
+    local domain num requestedListname existingTypeId existingListname
     domain="$1"
-    list="$2"
 
     # Is the domain in the list we want to add it to?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}';")"
+    requestedListname="$(GetListnameFromTypeId "${typeId}")"
 
     if [[ "${num}" -ne 0 ]]; then
-      if [[ "${verbose}" == true ]]; then
-          echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
+      existingTypeId="$(sqlite3 "${gravityDBfile}" "SELECT type FROM domainlist WHERE domain = '${domain}';")"
+      if [[ "${existingTypeId}" == "${typeId}" ]]; then
+        if [[ "${verbose}" == true ]]; then
+            echo -e "  ${INFO} ${1} already exists in ${requestedListname}, no need to add!"
+        fi
+      else
+        existingListname="$(GetListnameFromTypeId "${existingTypeId}")"
+        sqlite3 "${gravityDBfile}" "UPDATE domainlist SET type = ${typeId} WHERE domain='${domain}';"
+        if [[ "${verbose}" == true ]]; then
+            echo -e "  ${INFO} ${1} already exists in ${existingListname}, it has been moved to ${requestedListname}!"
+        fi
       fi
       return
     fi
 
     # Domain not found in the table, add it!
     if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} Adding ${1} to the ${listname}..."
+        echo -e "  ${INFO} Adding ${domain} to the ${requestedListname}..."
     fi
     reload=true
     # Insert only the domain here. The enabled and date_added fields will be filled
     # with their default values (enabled = true, date_added = current timestamp)
-    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES ('${domain}');"
+    sqlite3 "${gravityDBfile}" "INSERT INTO domainlist (domain,type) VALUES ('${domain}',${typeId});"
 }
 
 RemoveDomain() {
-    local domain list num
-    # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
+    local domain num requestedListname
     domain="$1"
-    list="$2"
 
     # Is the domain in the list we want to remove it from?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};")"
+
+    requestedListname="$(GetListnameFromTypeId "${typeId}")"
 
     if [[ "${num}" -eq 0 ]]; then
       if [[ "${verbose}" == true ]]; then
-          echo -e "  ${INFO} ${1} does not exist in ${list}, no need to remove!"
+          echo -e "  ${INFO} ${domain} does not exist in ${requestedListname}, no need to remove!"
       fi
       return
     fi
 
     # Domain found in the table, remove it!
     if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} Removing ${1} from the ${listname}..."
+        echo -e "  ${INFO} Removing ${domain} from the ${requestedListname}..."
     fi
     reload=true
     # Remove it from the current list
-    sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = '${domain}';"
+    sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE domain = '${domain}' AND type = ${typeId};"
 }
 
 Displaylist() {
-    local list listname count num_pipes domain enabled status nicedate
+    local count num_pipes domain enabled status nicedate requestedListname
 
-    listname="${listType}"
-    data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)"
+    requestedListname="$(GetListnameFromTypeId "${typeId}")"
+    data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM domainlist WHERE type = ${typeId};" 2> /dev/null)"
 
     if [[ -z $data ]]; then
         echo -e "Not showing empty list"
     else
-        echo -e "Displaying ${listname}:"
+        echo -e "Displaying ${requestedListname}:"
         count=1
         while IFS= read -r line
         do
@@ -221,17 +221,17 @@ Displaylist() {
 }
 
 NukeList() {
-    sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};"
+    sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};"
 }
 
 for var in "$@"; do
     case "${var}" in
-        "-w" | "whitelist"   ) listType="whitelist"; listAlt="blacklist";;
-        "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
-        "--wild" | "wildcard" ) listType="regex_blacklist"; wildcard=true;;
-        "--regex" | "regex"   ) listType="regex_blacklist";;
-        "--white-regex" | "white-regex" ) listType="regex_whitelist";;
-        "--white-wild" | "white-wild" ) listType="regex_whitelist"; wildcard=true;;
+        "-w" | "whitelist"   ) typeId=0;;
+        "-b" | "blacklist"   ) typeId=1;;
+        "--white-regex" | "white-regex" ) typeId=2;;
+        "--white-wild" | "white-wild" ) typeId=2; wildcard=true;;
+        "--wild" | "wildcard" ) typeId=3; wildcard=true;;
+        "--regex" | "regex"   ) typeId=3;;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;
@@ -239,7 +239,7 @@ for var in "$@"; do
         "-l" | "--list"      ) Displaylist;;
         "--nuke"             ) NukeList;;
         "--web"              ) web=true;;
-        *                    ) HandleOther "${var}";;
+        *                    ) ValidateDomain "${var}";;
     esac
 done
 
diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh
index 673ded0b..31009dd9 100644
--- a/advanced/Scripts/piholeCheckout.sh
+++ b/advanced/Scripts/piholeCheckout.sh
@@ -46,6 +46,12 @@ checkout() {
     local corebranches
     local webbranches
 
+    # Check if FTL is installed - do this early on as FTL is a hard dependency for Pi-hole
+    local funcOutput
+    funcOutput=$(get_binary_name) #Store output of get_binary_name here
+    local binary
+    binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL)
+
     # Avoid globbing
     set -f
 
@@ -86,7 +92,6 @@ checkout() {
         fi
         #echo -e "  ${TICK} Pi-hole Core"
 
-        get_binary_name
         local path
         path="development/${binary}"
         echo "development" > /etc/pihole/ftlbranch
@@ -101,7 +106,6 @@ checkout() {
             fetch_checkout_pull_branch "${webInterfaceDir}" "master" || { echo "  ${CROSS} Unable to pull Web master branch"; exit 1; }
         fi
         #echo -e "  ${TICK} Web Interface"
-        get_binary_name
         local path
         path="master/${binary}"
         echo "master" > /etc/pihole/ftlbranch
@@ -161,7 +165,6 @@ checkout() {
         fi
         checkout_pull_branch "${webInterfaceDir}" "${2}"
     elif [[ "${1}" == "ftl" ]] ; then
-        get_binary_name
         local path
         path="${2}/${binary}"
 
diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh
index 4d352777..e45be5cf 100755
--- a/advanced/Scripts/update.sh
+++ b/advanced/Scripts/update.sh
@@ -31,7 +31,6 @@ source "/opt/pihole/COL_TABLE"
 # make_repo() sourced from basic-install.sh
 # update_repo() source from basic-install.sh
 # getGitFiles() sourced from basic-install.sh
-# get_binary_name() sourced from basic-install.sh
 # FTLcheckUpdate() sourced from basic-install.sh
 
 GitCheckUpdateAvail() {
@@ -129,7 +128,12 @@ main() {
         fi
     fi
 
-    if FTLcheckUpdate > /dev/null; then
+    local funcOutput
+    funcOutput=$(get_binary_name) #Store output of get_binary_name here
+    local binary
+    binary="pihole-FTL${funcOutput##*pihole-FTL}" #binary name will be the last line of the output of get_binary_name (it always begins with pihole-FTL)
+
+    if FTLcheckUpdate "${binary}" > /dev/null; then
         FTL_update=true
         echo -e "  ${INFO} FTL:\\t\\t${COL_YELLOW}update available${COL_NC}"
     else
diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index 411cc1f6..829ba57b 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -17,6 +17,7 @@ readonly FTLconf="/etc/pihole/pihole-FTL.conf"
 # 03 -> wildcards
 readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf"
 readonly PI_HOLE_BIN_DIR="/usr/local/bin"
+readonly dnscustomfile="/etc/pihole/custom.list"
 
 readonly gravityDBfile="/etc/pihole/gravity.db"
 
@@ -404,13 +405,15 @@ SetWebUILayout() {
 CustomizeAdLists() {
     local address
     address="${args[3]}"
+    local comment
+    comment="${args[4]}"
 
     if [[ "${args[2]}" == "enable" ]]; then
         sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 1 WHERE address = '${address}'"
     elif [[ "${args[2]}" == "disable" ]]; then
         sqlite3 "${gravityDBfile}" "UPDATE adlist SET enabled = 0 WHERE address = '${address}'"
     elif [[ "${args[2]}" == "add" ]]; then
-        sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address) VALUES ('${address}')"
+        sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlist (address, comment) VALUES ('${address}', '${comment}')"
     elif [[ "${args[2]}" == "del" ]]; then
         sqlite3 "${gravityDBfile}" "DELETE FROM adlist WHERE address = '${address}'"
     else
@@ -524,10 +527,10 @@ Interfaces:
   fi
 
     if [[ "${args[2]}" == "all" ]]; then
-        echo -e "  ${INFO} Listening on all interfaces, permiting all origins. Please use a firewall!"
+        echo -e "  ${INFO} Listening on all interfaces, permitting all origins. Please use a firewall!"
         change_setting "DNSMASQ_LISTENING" "all"
     elif [[ "${args[2]}" == "local" ]]; then
-        echo -e "  ${INFO} Listening on all interfaces, permiting origins from one hop away (LAN)"
+        echo -e "  ${INFO} Listening on all interfaces, permitting origins from one hop away (LAN)"
         change_setting "DNSMASQ_LISTENING" "local"
     else
         echo -e "  ${INFO} Listening only on interface ${PIHOLE_INTERFACE}"
@@ -597,6 +600,28 @@ SetPrivacyLevel() {
     fi
 }
 
+AddCustomDNSAddress() {
+    echo -e "  ${TICK} Adding custom DNS entry..."
+
+    ip="${args[2]}"
+    host="${args[3]}"
+	echo "${ip} ${host}" >> "${dnscustomfile}"
+
+    # Restart dnsmasq to load new custom DNS entries
+    RestartDNS
+}
+
+RemoveCustomDNSAddress() {
+    echo -e "  ${TICK} Removing custom DNS entry..."
+
+    ip="${args[2]}"
+    host="${args[3]}"
+    sed -i "/${ip} ${host}/d" "${dnscustomfile}"
+
+    # Restart dnsmasq to update removed custom DNS entries
+    RestartDNS
+}
+
 main() {
     args=("$@")
 
@@ -628,6 +653,8 @@ main() {
         "audit"               ) addAudit "$@";;
         "clearaudit"          ) clearAudit;;
         "-l" | "privacylevel" ) SetPrivacyLevel;;
+        "addcustomdns"        ) AddCustomDNSAddress;;
+        "removecustomdns"     ) RemoveCustomDNSAddress;;
         *                     ) helpFunc;;
     esac
 
diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh
index 8c9578a3..b4b6b4a1 100644
--- a/advanced/Scripts/wildcard_regex_converter.sh
+++ b/advanced/Scripts/wildcard_regex_converter.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 # Pi-hole: A black hole for Internet advertisements
 # (c) 2017 Pi-hole, LLC (https://pi-hole.net)
 # Network-wide ad blocking via your own hardware.
diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service
index 8a4c7ce6..5dbf080e 100644
--- a/advanced/Templates/pihole-FTL.service
+++ b/advanced/Templates/pihole-FTL.service
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
 ### BEGIN INIT INFO
 # Provides:          pihole-FTL
 # Required-Start:    $remote_fs $syslog