Improve gravity performance (#3100)

* Gravity performance improvements.

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Do not move downloaded lists into migration_backup directory.

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Do not (strictly) sort domains. Random-leaf access is faster than always-last-leaf access (on average).

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Append instead of overwrite gravity_new collection list.

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Rename table gravity_new to gravity_temp to clarify that this is only an intermediate table.

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Add timers for each of the calls to compute intense parts. They are to be removed before this finally hits the release/v5.0 branch.

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Fix legacy list files import. It currently doesn't work when the gravity database has already been updated to using the single domainlist table.

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Simplify database_table_from_file(), remove all to this function for gravity lost downloads.

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Update gravity.db.sql to version 10 to have newle created databases already reflect the most recent state.

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Create second gravity database and swap them on success. This has a number of advantages such as instantaneous gravity updates (as seen from FTL) and always available gravity blocking. Furthermore, this saves disk space as the old database is removed on completion.

* Add timing output for the database swapping SQLite3 call.

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Explicitly generate index as a separate process.

Signed-off-by: DL6ER <dl6er@dl6er.de>

* Remove time measurements.

Signed-off-by: DL6ER <dl6er@dl6er.de>

2 files changed, 135 insertions, 76 deletions

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index d0c744f4..a7dc12df 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,16 +1,21 @@
-PRAGMA FOREIGN_KEYS=ON;
+PRAGMA foreign_keys=OFF;
+BEGIN TRANSACTION;
 
 CREATE TABLE "group"
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
 	enabled BOOLEAN NOT NULL DEFAULT 1,
-	name TEXT NOT NULL,
+	name TEXT UNIQUE NOT NULL,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	description TEXT
 );
+INSERT INTO "group" (id,enabled,name) VALUES (0,1,'Unassociated');
 
-CREATE TABLE whitelist
+CREATE TABLE domainlist
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	type INTEGER NOT NULL DEFAULT 0,
 	domain TEXT UNIQUE NOT NULL,
 	enabled BOOLEAN NOT NULL DEFAULT 1,
 	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
@@ -18,125 +23,158 @@ CREATE TABLE whitelist
 	comment TEXT
 );
 
-CREATE TABLE whitelist_by_group
-(
-	whitelist_id INTEGER NOT NULL REFERENCES whitelist (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (whitelist_id, group_id)
-);
-
-CREATE TABLE blacklist
+CREATE TABLE adlist
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	domain TEXT UNIQUE NOT NULL,
+	address TEXT UNIQUE NOT NULL,
 	enabled BOOLEAN NOT NULL DEFAULT 1,
 	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
 	comment TEXT
 );
 
-CREATE TABLE blacklist_by_group
+CREATE TABLE adlist_by_group
 (
-	blacklist_id INTEGER NOT NULL REFERENCES blacklist (id),
+	adlist_id INTEGER NOT NULL REFERENCES adlist (id),
 	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (blacklist_id, group_id)
+	PRIMARY KEY (adlist_id, group_id)
 );
 
-CREATE TABLE regex
+CREATE TABLE gravity
 (
-	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	domain TEXT UNIQUE NOT NULL,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
+	domain TEXT NOT NULL,
+	adlist_id INTEGER NOT NULL REFERENCES adlist (id)
 );
 
-CREATE TABLE regex_by_group
+CREATE TABLE info
 (
-	regex_id INTEGER NOT NULL REFERENCES regex (id),
-	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (regex_id, group_id)
+	property TEXT PRIMARY KEY,
+	value TEXT NOT NULL
 );
 
-CREATE TABLE adlist
+INSERT INTO "info" VALUES('version','10');
+
+CREATE TABLE domain_audit
 (
 	id INTEGER PRIMARY KEY AUTOINCREMENT,
-	address TEXT UNIQUE NOT NULL,
-	enabled BOOLEAN NOT NULL DEFAULT 1,
-	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-	comment TEXT
+	domain TEXT UNIQUE NOT NULL,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int))
 );
 
-CREATE TABLE adlist_by_group
+CREATE TABLE domainlist_by_group
 (
-	adlist_id INTEGER NOT NULL REFERENCES adlist (id),
+	domainlist_id INTEGER NOT NULL REFERENCES domainlist (id),
 	group_id INTEGER NOT NULL REFERENCES "group" (id),
-	PRIMARY KEY (adlist_id, group_id)
+	PRIMARY KEY (domainlist_id, group_id)
 );
 
-CREATE TABLE gravity
+CREATE TABLE client
 (
-	domain TEXT PRIMARY KEY
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	ip TEXT NOL NULL UNIQUE
 );
 
-CREATE TABLE info
+CREATE TABLE client_by_group
 (
-	property TEXT PRIMARY KEY,
-	value TEXT NOT NULL
+	client_id INTEGER NOT NULL REFERENCES client (id),
+	group_id INTEGER NOT NULL REFERENCES "group" (id),
+	PRIMARY KEY (client_id, group_id)
 );
 
-INSERT INTO info VALUES("version","1");
+CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist
+    BEGIN
+      UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address;
+    END;
+
+CREATE TRIGGER tr_domainlist_update AFTER UPDATE ON domainlist
+    BEGIN
+      UPDATE domainlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+    END;
+
+CREATE VIEW vw_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 0
+    ORDER BY domainlist.id;
+
+CREATE VIEW vw_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 1
+    ORDER BY domainlist.id;
+
+CREATE VIEW vw_regex_whitelist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 2
+    ORDER BY domainlist.id;
+
+CREATE VIEW vw_regex_blacklist AS SELECT domain, domainlist.id AS id, domainlist_by_group.group_id AS group_id
+    FROM domainlist
+    LEFT JOIN domainlist_by_group ON domainlist_by_group.domainlist_id = domainlist.id
+    LEFT JOIN "group" ON "group".id = domainlist_by_group.group_id
+    WHERE domainlist.enabled = 1 AND (domainlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    AND domainlist.type = 3
+    ORDER BY domainlist.id;
+
+CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id
+    FROM gravity
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = gravity.adlist_id
+    LEFT JOIN adlist ON adlist.id = gravity.adlist_id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1);
+
+CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id
+    FROM adlist
+    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id
+    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
+    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1)
+    ORDER BY adlist.id;
 
-CREATE VIEW vw_whitelist AS SELECT DISTINCT domain
-    FROM whitelist
-    LEFT JOIN whitelist_by_group ON whitelist_by_group.whitelist_id = whitelist.id
-    LEFT JOIN "group" ON "group".id = whitelist_by_group.group_id
-    WHERE whitelist.enabled = 1 AND (whitelist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY whitelist.id;
+CREATE TRIGGER tr_domainlist_add AFTER INSERT ON domainlist
+    BEGIN
+      INSERT INTO domainlist_by_group (domainlist_id, group_id) VALUES (NEW.id, 0);
+    END;
 
-CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
+CREATE TRIGGER tr_client_add AFTER INSERT ON client
     BEGIN
-      UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+      INSERT INTO client_by_group (client_id, group_id) VALUES (NEW.id, 0);
     END;
 
-CREATE VIEW vw_blacklist AS SELECT DISTINCT domain
-    FROM blacklist
-    LEFT JOIN blacklist_by_group ON blacklist_by_group.blacklist_id = blacklist.id
-    LEFT JOIN "group" ON "group".id = blacklist_by_group.group_id
-    WHERE blacklist.enabled = 1 AND (blacklist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY blacklist.id;
+CREATE TRIGGER tr_adlist_add AFTER INSERT ON adlist
+    BEGIN
+      INSERT INTO adlist_by_group (adlist_id, group_id) VALUES (NEW.id, 0);
+    END;
 
-CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
+CREATE TRIGGER tr_group_update AFTER UPDATE ON "group"
     BEGIN
-      UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+      UPDATE "group" SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE id = NEW.id;
     END;
 
-CREATE VIEW vw_regex AS SELECT DISTINCT domain
-    FROM regex
-    LEFT JOIN regex_by_group ON regex_by_group.regex_id = regex.id
-    LEFT JOIN "group" ON "group".id = regex_by_group.group_id
-    WHERE regex.enabled = 1 AND (regex_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY regex.id;
+CREATE TRIGGER tr_group_zero AFTER DELETE ON "group"
+    BEGIN
+      INSERT OR IGNORE INTO "group" (id,enabled,name) VALUES (0,1,'Unassociated');
+    END;
 
-CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
+CREATE TRIGGER tr_domainlist_delete AFTER DELETE ON domainlist
     BEGIN
-      UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+      DELETE FROM domainlist_by_group WHERE domainlist_id = OLD.id;
     END;
 
-CREATE VIEW vw_adlist AS SELECT DISTINCT address
-    FROM adlist
-    LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id
-    LEFT JOIN "group" ON "group".id = adlist_by_group.group_id
-    WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1)
-    ORDER BY adlist.id;
+CREATE TRIGGER tr_adlist_delete AFTER DELETE ON adlist
+    BEGIN
+      DELETE FROM adlist_by_group WHERE adlist_id = OLD.id;
+    END;
 
-CREATE TRIGGER tr_adlist_update AFTER UPDATE ON adlist
+CREATE TRIGGER tr_client_delete AFTER DELETE ON client
     BEGIN
-      UPDATE adlist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address;
+      DELETE FROM client_by_group WHERE client_id = OLD.id;
     END;
 
-CREATE VIEW vw_gravity AS SELECT domain
-    FROM gravity
-    WHERE domain NOT IN (SELECT domain from vw_whitelist);
+COMMIT;
diff --git a/advanced/Templates/gravity_copy.sql b/advanced/Templates/gravity_copy.sql
new file mode 100644
index 00000000..e14d9d8c
--- /dev/null
+++ b/advanced/Templates/gravity_copy.sql
@@ -0,0 +1,21 @@
+.timeout 30000
+
+ATTACH DATABASE '/etc/pihole/gravity.db' AS OLD;
+
+BEGIN TRANSACTION;
+
+INSERT OR REPLACE INTO "group" SELECT * FROM OLD."group";
+INSERT OR REPLACE INTO domain_audit SELECT * FROM OLD.domain_audit;
+
+INSERT OR REPLACE INTO domainlist SELECT * FROM OLD.domainlist;
+INSERT OR REPLACE INTO domainlist_by_group SELECT * FROM OLD.domainlist_by_group;
+
+INSERT OR REPLACE INTO adlist SELECT * FROM OLD.adlist;
+INSERT OR REPLACE INTO adlist_by_group SELECT * FROM OLD.adlist_by_group;
+
+INSERT OR REPLACE INTO info SELECT * FROM OLD.info;
+
+INSERT OR REPLACE INTO client SELECT * FROM OLD.client;
+INSERT OR REPLACE INTO client_by_group SELECT * FROM OLD.client_by_group;
+
+COMMIT;