GeoIP2 implementation as a plugin (#12699)

* require geoip2/geoip2 composer package

* Determine region name based on Location Provider

* Adds empty GeoIp2 plugin

* move location_region column definition to GeoIp2 plugin
so it's get changed as soon as the plugin is activated

* Adds GeoIP2 location providers

* ignore GeoIP2 mmdb files

* Adds script to generate GeoIP2 test databases

* Adds Command to convert region codes from FIPS to ISO for old log table records

* Adds GeoIP2 AutoUpdater

* Use GeoIP2 in tests

* update test files

* code fixes

* adds tests

* rename old GeoIP providers to Legacy

* Let GeoIP autoupdater UI handle GeoIp2 as well

* convert region codes to ISO in API after switch to GeoIP2

* do not show GeoIP providers if GeoIP2 plugin is enabled an no GeoIP Legacy provider is still in use

* small fixes

* review changes

* Use correct region translations

* Show correct message if no database can be found

* if log tables have been converted, use archive date to check if region codes still need to be converted to iso

* fix tests

* Improves extracting GeoIP2 databases

* Adjust GeoLocation diagnostics

* readds old taiwan fixes

* Assume all third party location providers as 'recommended'

* Download database over HTTPS

* remove outdated comment (see https://github.com/matomo-org/matomo/issues/12411)

* Remove indication that Geoip2 may be slow, since we found it should be quite fast and this should not be an issue in theory

* skip detection if IP is empty & do not try convert IP to IPv4

* remove downloaded file if an error occurs while extracting

* command should be runnable multiple times

* use ISO codes for suggested region codes

* reload after wizard success

* Drop table if exists.

* Fix two translation keys.

* add special region handling for UK

* update system test

* update UI files

* submodule update

* update test files

5 files changed, 35 insertions, 38 deletions

diff --git a/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php b/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php
index 8dea041717..0a9bf260de 100644
--- a/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php
+++ b/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php
@@ -7,9 +7,9 @@
  */
 namespace Piwik\Tests\Fixtures;
 
+use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2;
 use Piwik\Plugins\Goals\API as APIGoals;
 use Piwik\Plugins\SegmentEditor\API as APISegmentEditor;
-use Piwik\Plugins\UserCountry\LocationProvider\GeoIp;
 use Piwik\Plugins\UserCountry\LocationProvider;
 use Piwik\Tests\Framework\Fixture;
 
@@ -35,11 +35,10 @@ class ManySitesImportedLogs extends Fixture
     public function setUp()
     {
         $this->setUpWebsitesAndGoals();
-        self::downloadGeoIpDbs();
 
         LocationProvider::$providers = null;
-        GeoIp::$geoIPDatabaseDir = 'tests/lib/geoip-files';
-        LocationProvider::setCurrentProvider('geoip_php');
+        GeoIp2::$geoIPDatabaseDir = 'tests/lib/geoip-files';
+        LocationProvider::setCurrentProvider('geoip2php');
 
         self::createSuperUser();
 
@@ -50,7 +49,7 @@ class ManySitesImportedLogs extends Fixture
     public function tearDown()
     {
         LocationProvider::$providers = null;
-        GeoIp::$geoIPDatabaseDir = 'tests/lib/geoip-files';
+        GeoIp2::$geoIPDatabaseDir = 'tests/lib/geoip-files';
         ManyVisitsWithGeoIP::unsetLocationProvider();
     }
 
diff --git a/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php b/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php
index 6c7cde409a..d687709eb9 100644
--- a/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php
+++ b/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php
@@ -9,8 +9,9 @@ namespace Piwik\Tests\Fixtures;
 
 use Piwik\Cache;
 use Piwik\Date;
+use Piwik\Option;
 use Piwik\Plugins\Goals\API;
-use Piwik\Plugins\UserCountry\LocationProvider\GeoIp;
+use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2;
 use Piwik\Plugins\UserCountry\LocationProvider;
 use Piwik\Tests\Framework\Fixture;
 use Exception;
@@ -25,7 +26,7 @@ require_once PIWIK_INCLUDE_PATH . '/tests/PHPUnit/Framework/Mock/LocationProvide
  */
 class ManyVisitsWithGeoIP extends Fixture
 {
-    const GEOIP_IMPL_TO_TEST = 'geoip_php';
+    const GEOIP_IMPL_TO_TEST = 'geoip2php';
 
     public $idSite = 1;
     public $dateTime = '2010-01-03 11:22:33';
@@ -35,10 +36,10 @@ class ManyVisitsWithGeoIP extends Fixture
         '::ffff:137.82.130.49', // in British Columbia (mapped ipv4)
         '137.82.130.0', // anonymization tests
         '137.82.0.0',
-        '2001:db8:85a3:0:0:8a2e:370:7334', // ipv6
+        '2003:f6:93bf:26f:9ec7:a6ff:fe29:27df', // ipv6 in US (without region or city)
         '113.62.1.1', // in Lhasa, Tibet
         '151.100.101.92', // in Rome, Italy (using country DB, so only Italy will show)
-        '103.29.196.229', // in Indonesia (Bali), (only Indonesia will show up)
+        '103.29.196.229', // in Indonesia, Central Java (Bali)
     );
 
     public $userAgents = array(
@@ -57,22 +58,24 @@ class ManyVisitsWithGeoIP extends Fixture
 
     public function setUp()
     {
+        // set option, so tracked data for the past won't get converted
+        Option::set(GeoIp2::SWITCH_TO_ISO_REGIONS_OPTION_NAME, 1);
+
         $this->setUpWebsitesAndGoals();
-        self::downloadGeoIpDbs();
 
         $this->setMockLocationProvider();
         $this->trackVisits(9, false);
 
-        $this->setLocationProvider('GeoIPCity.dat');
+        $this->setLocationProvider('GeoIP2-City.mmdb');
         $this->trackVisits(2, true, $useLocal = false);
         $this->trackVisits(4, true, $useLocal = false, $doBulk = true);
 
-        $this->setLocationProvider('GeoIP.dat');
+        $this->setLocationProvider('GeoIP2-Country.mmdb');
         $this->trackVisits(2, true);
 
         $this->trackOtherVisits();
 
-        $this->setLocationProvider('GeoIPCity.dat');
+        $this->setLocationProvider('GeoIP2-City.mmdb');
     }
 
     public function tearDown()
@@ -240,8 +243,8 @@ class ManyVisitsWithGeoIP extends Fixture
 
     public function setLocationProvider($file)
     {
-        GeoIp::$dbNames['loc'] = array($file);
-        GeoIp::$geoIPDatabaseDir = 'tests/lib/geoip-files';
+        GeoIp2::$dbNames['loc'] = array($file);
+        GeoIp2::$geoIPDatabaseDir = 'tests/lib/geoip-files';
         LocationProvider::$providers = null;
         LocationProvider::setCurrentProvider(self::GEOIP_IMPL_TO_TEST);
 
@@ -249,9 +252,9 @@ class ManyVisitsWithGeoIP extends Fixture
             throw new Exception("Failed to set the current location provider to '" . self::GEOIP_IMPL_TO_TEST . "'.");
         }
 
-        $possibleFiles = GeoIp::$dbNames['loc'];
-        if (GeoIp::getPathToGeoIpDatabase($possibleFiles) === false) {
-            throw new Exception("The GeoIP location provider cannot find the '$file' file! Tests will fail.");
+        $possibleFiles = GeoIp2::$dbNames['loc'];
+        if (GeoIp2::getPathToGeoIpDatabase($possibleFiles) === false) {
+            throw new Exception("The GeoIP2 location provider cannot find the '$file' file! Tests will fail.");
         }
     }
 
@@ -261,28 +264,28 @@ class ManyVisitsWithGeoIP extends Fixture
         LocationProvider::$providers[] = new MockLocationProvider();
         LocationProvider::setCurrentProvider('mock_provider');
         MockLocationProvider::$locations = array(
-            self::makeLocation('Stratford-upon-Avon', 'P3', 'gb', 123.456, 21.321), // template location
+            self::makeLocation('Stratford-upon-Avon', 'WAR', 'gb', 123.456, 21.321), // template location
 
             // same region, different city, same country
-            self::makeLocation('Nuneaton and Bedworth', 'P3', 'gb', $isp = 'comcast.net'),
+            self::makeLocation('Nuneaton and Bedworth', 'WAR', 'gb', $isp = 'comcast.net'),
 
             // same region, city & country (different lat/long)
-            self::makeLocation('Stratford-upon-Avon', 'P3', 'gb', 124.456, 22.231, $isp = 'comcast.net'),
+            self::makeLocation('Stratford-upon-Avon', 'WAR', 'gb', 124.456, 22.231, $isp = 'comcast.net'),
 
             // same country, different region & city
-            self::makeLocation('London', 'H9', 'gb'),
+            self::makeLocation('London', 'LND', 'gb'),
 
             // same country, different region, same city
-            self::makeLocation('Stratford-upon-Avon', 'G5', 'gb', $lat = null, $long = null, $isp = 'awesomeisp.com'),
+            self::makeLocation('Stratford-upon-Avon', 'KEN', 'gb', $lat = null, $long = null, $isp = 'awesomeisp.com'),
 
             // different country, diff region, same city
-            self::makeLocation('Stratford-upon-Avon', '66', 'ru'),
+            self::makeLocation('Stratford-upon-Avon', 'SPE', 'ru'),
 
             // different country, diff region (same as last), different city
-            self::makeLocation('Hluboká nad Vltavou', '66', 'ru'),
+            self::makeLocation('Hluboká nad Vltavou', 'SPE', 'ru'),
 
             // different country, diff region (same as last), same city
-            self::makeLocation('Stratford-upon-Avon', '66', 'mk'),
+            self::makeLocation('Stratford-upon-Avon', '18', 'mk'),
 
             // unknown location
             self::makeLocation(null, null, null),
@@ -297,4 +300,4 @@ class ManyVisitsWithGeoIP extends Fixture
             // ignore error
         }
     }
-}
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Fixtures/ManyVisitsWithMockLocationProvider.php b/tests/PHPUnit/Fixtures/ManyVisitsWithMockLocationProvider.php
index ae5b1ae4dd..fb3d412e09 100644
--- a/tests/PHPUnit/Fixtures/ManyVisitsWithMockLocationProvider.php
+++ b/tests/PHPUnit/Fixtures/ManyVisitsWithMockLocationProvider.php
@@ -218,11 +218,11 @@ class ManyVisitsWithMockLocationProvider extends Fixture
         MockLocationProvider::$locations = array(
             self::makeLocation('Toronto', 'ON', 'CA', $lat = null, $long = null, $isp = 'comcast.net'),
 
-            self::makeLocation('Nice', 'B8', 'FR', $lat = null, $long = null, $isp = 'comcast.net'),
+            self::makeLocation('Nice', 'PAC', 'FR', $lat = null, $long = null, $isp = 'comcast.net'),
 
-            self::makeLocation('Melbourne', '07', 'AU', $lat = null, $long = null, $isp = 'awesomeisp.com'),
+            self::makeLocation('Melbourne', 'VIC', 'AU', $lat = null, $long = null, $isp = 'awesomeisp.com'),
 
-            self::makeLocation('Yokohama', '19', 'JP'),
+            self::makeLocation('Yokohama', '14', 'JP'),
         );
     }
 }
\ No newline at end of file
diff --git a/tests/PHPUnit/Fixtures/SqlDump.php b/tests/PHPUnit/Fixtures/SqlDump.php
index f5ebc7b364..8069c4b67e 100644
--- a/tests/PHPUnit/Fixtures/SqlDump.php
+++ b/tests/PHPUnit/Fixtures/SqlDump.php
@@ -85,8 +85,6 @@ class SqlDump extends Fixture
     }
 
     /**
-     * maybe this could use downloadAndUnzip(self::$geoLiteCityDbUrl, $geoIpOutputDir, 'GeoIPCity.dat');
-     *
      * @param $dumpPath
      * @return int
      */
diff --git a/tests/PHPUnit/Fixtures/UITestFixture.php b/tests/PHPUnit/Fixtures/UITestFixture.php
index 403abac8d5..b7a316f1f6 100644
--- a/tests/PHPUnit/Fixtures/UITestFixture.php
+++ b/tests/PHPUnit/Fixtures/UITestFixture.php
@@ -9,8 +9,6 @@ namespace Piwik\Tests\Fixtures;
 
 use Exception;
 use Piwik\API\Request;
-use Piwik\AssetManager;
-use Piwik\Access;
 use Piwik\Common;
 use Piwik\Date;
 use Piwik\Db;
@@ -18,12 +16,12 @@ use Piwik\DbHelper;
 use Piwik\FrontController;
 use Piwik\Option;
 use Piwik\Piwik;
+use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2;
 use Piwik\Plugins\PrivacyManager\IPAnonymizer;
 use Piwik\Plugins\SegmentEditor\API as APISegmentEditor;
 use Piwik\Plugins\UserCountry\LocationProvider;
 use Piwik\Plugins\UsersManager\API as UsersManagerAPI;
 use Piwik\Plugins\SitesManager\API as SitesManagerAPI;
-use Piwik\Tests\Framework\Fixture;
 use Piwik\Plugins\VisitsSummary\API as VisitsSummaryAPI;
 use Piwik\Config as PiwikConfig;
 
@@ -42,8 +40,6 @@ class UITestFixture extends SqlDump
 
     public function setUp()
     {
-        self::downloadGeoIpDbs();
-
         parent::setUp();
 
         self::resetPluginsInstalledConfig();
@@ -57,7 +53,8 @@ class UITestFixture extends SqlDump
         );
 
         // for proper geolocation
-        LocationProvider::setCurrentProvider(LocationProvider\GeoIp\Php::ID);
+        GeoIp2::$geoIPDatabaseDir = 'tests/lib/geoip-files';
+        LocationProvider::setCurrentProvider(GeoIp2\Php::ID);
         IPAnonymizer::deactivate();
 
         $this->addOverlayVisits();