GeoIP2 implementation as a plugin (#12699)

* require geoip2/geoip2 composer package

* Determine region name based on Location Provider

* Adds empty GeoIp2 plugin

* move location_region column definition to GeoIp2 plugin
so it's get changed as soon as the plugin is activated

* Adds GeoIP2 location providers

* ignore GeoIP2 mmdb files

* Adds script to generate GeoIP2 test databases

* Adds Command to convert region codes from FIPS to ISO for old log table records

* Adds GeoIP2 AutoUpdater

* Use GeoIP2 in tests

* update test files

* code fixes

* adds tests

* rename old GeoIP providers to Legacy

* Let GeoIP autoupdater UI handle GeoIp2 as well

* convert region codes to ISO in API after switch to GeoIP2

* do not show GeoIP providers if GeoIP2 plugin is enabled an no GeoIP Legacy provider is still in use

* small fixes

* review changes

* Use correct region translations

* Show correct message if no database can be found

* if log tables have been converted, use archive date to check if region codes still need to be converted to iso

* fix tests

* Improves extracting GeoIP2 databases

* Adjust GeoLocation diagnostics

* readds old taiwan fixes

* Assume all third party location providers as 'recommended'

* Download database over HTTPS

* remove outdated comment (see https://github.com/matomo-org/matomo/issues/12411)

* Remove indication that Geoip2 may be slow, since we found it should be quite fast and this should not be an issue in theory

* skip detection if IP is empty & do not try convert IP to IPv4

* remove downloaded file if an error occurs while extracting

* command should be runnable multiple times

* use ISO codes for suggested region codes

* reload after wizard success

* Drop table if exists.

* Fix two translation keys.

* add special region handling for UK

* update system test

* update UI files

* submodule update

* update test files

1 files changed, 0 insertions, 71 deletions

diff --git a/tests/PHPUnit/Framework/Fixture.php b/tests/PHPUnit/Framework/Fixture.php
index a64f8b6f01..efec12237e 100644
--- a/tests/PHPUnit/Framework/Fixture.php
+++ b/tests/PHPUnit/Framework/Fixture.php
@@ -809,77 +809,6 @@ class Fixture extends \PHPUnit_Framework_Assert
             strpos( $gdInfo['GD Version'], self::IMAGES_GENERATED_FOR_GD) !== false;
     }
 
-    public static $geoIpDbUrl = 'http://piwik-team.s3.amazonaws.com/GeoIP.dat.gz';
-    public static $geoLiteCityDbUrl = 'http://piwik-team.s3.amazonaws.com/GeoLiteCity.dat.gz';
-
-    public static function downloadGeoIpDbs()
-    {
-        $geoIpOutputDir = PIWIK_INCLUDE_PATH . '/tests/lib/geoip-files';
-        self::downloadAndUnzip(self::$geoIpDbUrl, $geoIpOutputDir, 'GeoIP.dat');
-        self::downloadAndUnzip(self::$geoLiteCityDbUrl, $geoIpOutputDir, 'GeoIPCity.dat');
-    }
-
-    public static function downloadAndUnzip($url, $outputDir, $filename)
-    {
-        $bufferSize = 1024 * 1024;
-
-        if (!is_dir($outputDir)) {
-            mkdir($outputDir);
-        }
-
-        $deflatedOut = $outputDir . '/' . $filename;
-        $outfileName = $deflatedOut . '.gz';
-
-        if (file_exists($deflatedOut)) {
-            $filesize = filesize($deflatedOut);
-            if($filesize == 0) {
-                throw new Exception("The file $deflatedOut is empty. Suggestion: delete it and try again.");
-            }
-
-            self::copyDownloadedGeoIp($deflatedOut, $filename);
-
-            // Valid geoip db found
-            return;
-        }
-
-        echo "Geoip database $outfileName is not found. Downloading from $url...\n";
-
-        $dump = fopen($url, 'rb');
-        if($dump === false){
-            throw new Exception('Could not download Geoip database from ' . $url);
-        }
-        
-        $outfile = fopen($outfileName, 'wb');
-        if(!$outfile) {
-            throw new Exception("Failed to create file $outfileName - please check permissions");
-        }
-
-        while (!feof($dump)) {
-            fwrite($outfile, fread($dump, $bufferSize), $bufferSize);
-        }
-        fclose($dump);
-        fclose($outfile);
-
-        // unzip the dump
-        exec("gunzip -c \"" . $outfileName . "\" > \"$deflatedOut\"", $output, $return);
-        if ($return !== 0) {
-            Log::info("gunzip failed with file that has following contents:");
-            Log::info(file_get_contents($outfile));
-
-            throw new Exception("gunzip failed($return): " . implode("\n", $output));
-        }
-
-        self::copyDownloadedGeoIp($deflatedOut, $filename);
-    }
-
-    private static function copyDownloadedGeoIp($deflatedOut, $filename)
-    {
-        $realFileOut = PIWIK_INCLUDE_PATH . '/' . LocationProvider\GeoIp::$geoIPDatabaseDir . '/' . $filename;
-        if (!file_exists($realFileOut)) {
-            copy($deflatedOut, $realFileOut);
-        }
-    }
-
     public static function executeLogImporter($logFile, $options, $allowFailure = false)
     {
         $python = self::getPythonBinary();