diff options
author | DL6ER <DL6ER@users.noreply.github.com> | 2021-02-28 19:01:19 +0300 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-02-28 19:01:19 +0300 |
commit | 57fe3b6621cc965cfec7dfa25510f7b268dda476 (patch) | |
tree | 5fa8da1cb57584d51d2a5d2f9a4e058ba9aefe15 /gravity.sh | |
parent | 56fa9714b544ce0989963ed52cb4af6456661732 (diff) | |
parent | a2625df5e2a7e406cadcd430ea3902c47234769c (diff) |
Merge pull request #3951 from pi-hole/new/gravity_adlist_infos
Store gravity details in gravity.db adlist table
Diffstat (limited to 'gravity.sh')
-rwxr-xr-x | gravity.sh | 99 |
1 files changed, 92 insertions, 7 deletions
@@ -217,6 +217,48 @@ database_adlist_updated() { fi } +# Check if a column with name ${2} exists in gravity table with name ${1} +gravity_column_exists() { + output=$( { printf ".timeout 30000\\nSELECT EXISTS(SELECT * FROM pragma_table_info('%s') WHERE name='%s');\\n" "${1}" "${2}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + if [[ "${output}" == "1" ]]; then + return 0 # Bash 0 is success + fi + + return 1 # Bash non-0 is failure +} + +# Update number of domain on this list. We store this in the "old" database as all values in the new database will later be overwritten +database_adlist_number() { + # Only try to set number of domains when this field exists in the gravity database + if ! gravity_column_exists "adlist" "number"; then + return; + fi + + output=$( { printf ".timeout 30000\\nUPDATE adlist SET number = %i, invalid_domains = %i WHERE id = %i;\\n" "${num_lines}" "${num_invalid}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to update number of domains in adlist with ID ${1} in database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi +} + +# Update status of this list. We store this in the "old" database as all values in the new database will later be overwritten +database_adlist_status() { + # Only try to set the status when this field exists in the gravity database + if ! gravity_column_exists "adlist" "status"; then + return; + fi + + output=$( { printf ".timeout 30000\\nUPDATE adlist SET status = %i WHERE id = %i;\\n" "${2}" "${1}" | sqlite3 "${gravityDBfile}"; } 2>&1 ) + status="$?" + + if [[ "${status}" -ne 0 ]]; then + echo -e "\\n ${CROSS} Unable to update status of adlist with ID ${1} in database ${gravityDBfile}\\n ${output}" + gravity_Cleanup "error" + fi +} + # Migrate pre-v5.0 list files to database-based Pi-hole versions migrate_to_database() { # Create database file only if not present @@ -444,6 +486,8 @@ gravity_DownloadBlocklists() { } total_num=0 +num_lines=0 +num_invalid=0 parseList() { local adlistID="${1}" src="${2}" target="${3}" incorrect_lines # This sed does the following things: @@ -454,7 +498,7 @@ parseList() { # Find (up to) five domains containing invalid characters (see above) incorrect_lines="$(sed -e "/[^a-zA-Z0-9.\_-]/!d" "${src}" | head -n 5)" - local num_lines num_target_lines num_correct_lines num_invalid + local num_target_lines num_correct_lines num_invalid # Get number of lines in source file num_lines="$(grep -c "^" "${src}")" # Get number of lines in destination file @@ -463,9 +507,9 @@ parseList() { total_num="$num_target_lines" num_invalid="$(( num_lines-num_correct_lines ))" if [[ "${num_invalid}" -eq 0 ]]; then - echo " ${INFO} Received ${num_lines} domains" + echo " ${INFO} Analyzed ${num_lines} domains" else - echo " ${INFO} Received ${num_lines} domains, ${num_invalid} domains invalid!" + echo " ${INFO} Analyzed ${num_lines} domains, ${num_invalid} domains invalid!" fi # Display sample of invalid lines if we found some @@ -476,6 +520,29 @@ parseList() { done <<< "${incorrect_lines}" fi } +compareLists() { + local adlistID="${1}" target="${2}" + + # Verify checksum when an older checksum exists + if [[ -s "${target}.sha1" ]]; then + if ! sha1sum --check --status --strict "${target}.sha1"; then + # The list changed upstream, we need to update the checksum + sha1sum "${target}" > "${target}.sha1" + echo " ${INFO} List has been updated" + database_adlist_status "${adlistID}" "1" + database_adlist_updated "${adlistID}" + else + echo " ${INFO} List stayed unchanged" + database_adlist_status "${adlistID}" "2" + fi + else + # No checksum available, create one for comparing on the next run + sha1sum "${target}" > "${target}.sha1" + # We assume here it was changed upstream + database_adlist_status "${adlistID}" "1" + database_adlist_updated "${adlistID}" + fi +} # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { @@ -559,31 +626,49 @@ gravity_DownloadBlocklistFromUrl() { esac;; esac + local done="false" # Determine if the blocklist was downloaded and saved correctly if [[ "${success}" == true ]]; then if [[ "${httpCode}" == "304" ]]; then # Add domains to database table file parseList "${adlistID}" "${saveLocation}" "${target}" + database_adlist_status "${adlistID}" "2" + database_adlist_number "${adlistID}" + done="true" # Check if $patternbuffer is a non-zero length file elif [[ -s "${patternBuffer}" ]]; then # Determine if blocklist is non-standard and parse as appropriate gravity_ParseFileIntoDomains "${patternBuffer}" "${saveLocation}" # Add domains to database table file parseList "${adlistID}" "${saveLocation}" "${target}" - # Update date_updated field in gravity database table - database_adlist_updated "${adlistID}" + # Compare lists, are they identical? + compareLists "${adlistID}" "${saveLocation}" + # Update gravity database table (status and updated timestamp are set in + # compareLists) + database_adlist_number "${adlistID}" + done="true" else # Fall back to previously cached list if $patternBuffer is empty - echo -e " ${INFO} Received empty file: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" + echo -e " ${INFO} Received empty file" fi - else + fi + + # Do we need to fall back to a cached list (if available)? + if [[ "${done}" != "true" ]]; then # Determine if cached list has read permission if [[ -r "${saveLocation}" ]]; then echo -e " ${CROSS} List download failed: ${COL_LIGHT_GREEN}using previously cached list${COL_NC}" # Add domains to database table file parseList "${adlistID}" "${saveLocation}" "${target}" + database_adlist_number "${adlistID}" + database_adlist_status "${adlistID}" "3" else echo -e " ${CROSS} List download failed: ${COL_LIGHT_RED}no cached list available${COL_NC}" + # Manually reset these two numbers because we do not call parseList here + num_lines=0 + num_invalid=0 + database_adlist_number "${adlistID}" + database_adlist_status "${adlistID}" "4" fi fi } |