diff options
author | Stefan Giehl <stefan@piwik.org> | 2018-05-04 08:15:12 +0300 |
---|---|---|
committer | Matthieu Aubry <mattab@users.noreply.github.com> | 2018-05-04 08:15:12 +0300 |
commit | 18ad4f7f04d01b90fb9fc1d623585b5033092ed8 (patch) | |
tree | 256d6b3628c3d39b7d911f35698636d331194109 /tests/PHPUnit/Fixtures | |
parent | 453e3fbdabbf8c72264a3e15e2650cfad99b06b2 (diff) |
GeoIP2 implementation as a plugin (#12699)
* require geoip2/geoip2 composer package
* Determine region name based on Location Provider
* Adds empty GeoIp2 plugin
* move location_region column definition to GeoIp2 plugin
so it's get changed as soon as the plugin is activated
* Adds GeoIP2 location providers
* ignore GeoIP2 mmdb files
* Adds script to generate GeoIP2 test databases
* Adds Command to convert region codes from FIPS to ISO for old log table records
* Adds GeoIP2 AutoUpdater
* Use GeoIP2 in tests
* update test files
* code fixes
* adds tests
* rename old GeoIP providers to Legacy
* Let GeoIP autoupdater UI handle GeoIp2 as well
* convert region codes to ISO in API after switch to GeoIP2
* do not show GeoIP providers if GeoIP2 plugin is enabled an no GeoIP Legacy provider is still in use
* small fixes
* review changes
* Use correct region translations
* Show correct message if no database can be found
* if log tables have been converted, use archive date to check if region codes still need to be converted to iso
* fix tests
* Improves extracting GeoIP2 databases
* Adjust GeoLocation diagnostics
* readds old taiwan fixes
* Assume all third party location providers as 'recommended'
* Download database over HTTPS
* remove outdated comment (see https://github.com/matomo-org/matomo/issues/12411)
* Remove indication that Geoip2 may be slow, since we found it should be quite fast and this should not be an issue in theory
* skip detection if IP is empty & do not try convert IP to IPv4
* remove downloaded file if an error occurs while extracting
* command should be runnable multiple times
* use ISO codes for suggested region codes
* reload after wizard success
* Drop table if exists.
* Fix two translation keys.
* add special region handling for UK
* update system test
* update UI files
* submodule update
* update test files
Diffstat (limited to 'tests/PHPUnit/Fixtures')
-rw-r--r-- | tests/PHPUnit/Fixtures/ManySitesImportedLogs.php | 9 | ||||
-rw-r--r-- | tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php | 47 | ||||
-rw-r--r-- | tests/PHPUnit/Fixtures/ManyVisitsWithMockLocationProvider.php | 6 | ||||
-rw-r--r-- | tests/PHPUnit/Fixtures/SqlDump.php | 2 | ||||
-rw-r--r-- | tests/PHPUnit/Fixtures/UITestFixture.php | 9 |
5 files changed, 35 insertions, 38 deletions
diff --git a/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php b/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php index 8dea041717..0a9bf260de 100644 --- a/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php +++ b/tests/PHPUnit/Fixtures/ManySitesImportedLogs.php @@ -7,9 +7,9 @@ */ namespace Piwik\Tests\Fixtures; +use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2; use Piwik\Plugins\Goals\API as APIGoals; use Piwik\Plugins\SegmentEditor\API as APISegmentEditor; -use Piwik\Plugins\UserCountry\LocationProvider\GeoIp; use Piwik\Plugins\UserCountry\LocationProvider; use Piwik\Tests\Framework\Fixture; @@ -35,11 +35,10 @@ class ManySitesImportedLogs extends Fixture public function setUp() { $this->setUpWebsitesAndGoals(); - self::downloadGeoIpDbs(); LocationProvider::$providers = null; - GeoIp::$geoIPDatabaseDir = 'tests/lib/geoip-files'; - LocationProvider::setCurrentProvider('geoip_php'); + GeoIp2::$geoIPDatabaseDir = 'tests/lib/geoip-files'; + LocationProvider::setCurrentProvider('geoip2php'); self::createSuperUser(); @@ -50,7 +49,7 @@ class ManySitesImportedLogs extends Fixture public function tearDown() { LocationProvider::$providers = null; - GeoIp::$geoIPDatabaseDir = 'tests/lib/geoip-files'; + GeoIp2::$geoIPDatabaseDir = 'tests/lib/geoip-files'; ManyVisitsWithGeoIP::unsetLocationProvider(); } diff --git a/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php b/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php index 6c7cde409a..d687709eb9 100644 --- a/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php +++ b/tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php @@ -9,8 +9,9 @@ namespace Piwik\Tests\Fixtures; use Piwik\Cache; use Piwik\Date; +use Piwik\Option; use Piwik\Plugins\Goals\API; -use Piwik\Plugins\UserCountry\LocationProvider\GeoIp; +use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2; use Piwik\Plugins\UserCountry\LocationProvider; use Piwik\Tests\Framework\Fixture; use Exception; @@ -25,7 +26,7 @@ require_once PIWIK_INCLUDE_PATH . '/tests/PHPUnit/Framework/Mock/LocationProvide */ class ManyVisitsWithGeoIP extends Fixture { - const GEOIP_IMPL_TO_TEST = 'geoip_php'; + const GEOIP_IMPL_TO_TEST = 'geoip2php'; public $idSite = 1; public $dateTime = '2010-01-03 11:22:33'; @@ -35,10 +36,10 @@ class ManyVisitsWithGeoIP extends Fixture '::ffff:137.82.130.49', // in British Columbia (mapped ipv4) '137.82.130.0', // anonymization tests '137.82.0.0', - '2001:db8:85a3:0:0:8a2e:370:7334', // ipv6 + '2003:f6:93bf:26f:9ec7:a6ff:fe29:27df', // ipv6 in US (without region or city) '113.62.1.1', // in Lhasa, Tibet '151.100.101.92', // in Rome, Italy (using country DB, so only Italy will show) - '103.29.196.229', // in Indonesia (Bali), (only Indonesia will show up) + '103.29.196.229', // in Indonesia, Central Java (Bali) ); public $userAgents = array( @@ -57,22 +58,24 @@ class ManyVisitsWithGeoIP extends Fixture public function setUp() { + // set option, so tracked data for the past won't get converted + Option::set(GeoIp2::SWITCH_TO_ISO_REGIONS_OPTION_NAME, 1); + $this->setUpWebsitesAndGoals(); - self::downloadGeoIpDbs(); $this->setMockLocationProvider(); $this->trackVisits(9, false); - $this->setLocationProvider('GeoIPCity.dat'); + $this->setLocationProvider('GeoIP2-City.mmdb'); $this->trackVisits(2, true, $useLocal = false); $this->trackVisits(4, true, $useLocal = false, $doBulk = true); - $this->setLocationProvider('GeoIP.dat'); + $this->setLocationProvider('GeoIP2-Country.mmdb'); $this->trackVisits(2, true); $this->trackOtherVisits(); - $this->setLocationProvider('GeoIPCity.dat'); + $this->setLocationProvider('GeoIP2-City.mmdb'); } public function tearDown() @@ -240,8 +243,8 @@ class ManyVisitsWithGeoIP extends Fixture public function setLocationProvider($file) { - GeoIp::$dbNames['loc'] = array($file); - GeoIp::$geoIPDatabaseDir = 'tests/lib/geoip-files'; + GeoIp2::$dbNames['loc'] = array($file); + GeoIp2::$geoIPDatabaseDir = 'tests/lib/geoip-files'; LocationProvider::$providers = null; LocationProvider::setCurrentProvider(self::GEOIP_IMPL_TO_TEST); @@ -249,9 +252,9 @@ class ManyVisitsWithGeoIP extends Fixture throw new Exception("Failed to set the current location provider to '" . self::GEOIP_IMPL_TO_TEST . "'."); } - $possibleFiles = GeoIp::$dbNames['loc']; - if (GeoIp::getPathToGeoIpDatabase($possibleFiles) === false) { - throw new Exception("The GeoIP location provider cannot find the '$file' file! Tests will fail."); + $possibleFiles = GeoIp2::$dbNames['loc']; + if (GeoIp2::getPathToGeoIpDatabase($possibleFiles) === false) { + throw new Exception("The GeoIP2 location provider cannot find the '$file' file! Tests will fail."); } } @@ -261,28 +264,28 @@ class ManyVisitsWithGeoIP extends Fixture LocationProvider::$providers[] = new MockLocationProvider(); LocationProvider::setCurrentProvider('mock_provider'); MockLocationProvider::$locations = array( - self::makeLocation('Stratford-upon-Avon', 'P3', 'gb', 123.456, 21.321), // template location + self::makeLocation('Stratford-upon-Avon', 'WAR', 'gb', 123.456, 21.321), // template location // same region, different city, same country - self::makeLocation('Nuneaton and Bedworth', 'P3', 'gb', $isp = 'comcast.net'), + self::makeLocation('Nuneaton and Bedworth', 'WAR', 'gb', $isp = 'comcast.net'), // same region, city & country (different lat/long) - self::makeLocation('Stratford-upon-Avon', 'P3', 'gb', 124.456, 22.231, $isp = 'comcast.net'), + self::makeLocation('Stratford-upon-Avon', 'WAR', 'gb', 124.456, 22.231, $isp = 'comcast.net'), // same country, different region & city - self::makeLocation('London', 'H9', 'gb'), + self::makeLocation('London', 'LND', 'gb'), // same country, different region, same city - self::makeLocation('Stratford-upon-Avon', 'G5', 'gb', $lat = null, $long = null, $isp = 'awesomeisp.com'), + self::makeLocation('Stratford-upon-Avon', 'KEN', 'gb', $lat = null, $long = null, $isp = 'awesomeisp.com'), // different country, diff region, same city - self::makeLocation('Stratford-upon-Avon', '66', 'ru'), + self::makeLocation('Stratford-upon-Avon', 'SPE', 'ru'), // different country, diff region (same as last), different city - self::makeLocation('Hluboká nad Vltavou', '66', 'ru'), + self::makeLocation('Hluboká nad Vltavou', 'SPE', 'ru'), // different country, diff region (same as last), same city - self::makeLocation('Stratford-upon-Avon', '66', 'mk'), + self::makeLocation('Stratford-upon-Avon', '18', 'mk'), // unknown location self::makeLocation(null, null, null), @@ -297,4 +300,4 @@ class ManyVisitsWithGeoIP extends Fixture // ignore error } } -} +}
\ No newline at end of file diff --git a/tests/PHPUnit/Fixtures/ManyVisitsWithMockLocationProvider.php b/tests/PHPUnit/Fixtures/ManyVisitsWithMockLocationProvider.php index ae5b1ae4dd..fb3d412e09 100644 --- a/tests/PHPUnit/Fixtures/ManyVisitsWithMockLocationProvider.php +++ b/tests/PHPUnit/Fixtures/ManyVisitsWithMockLocationProvider.php @@ -218,11 +218,11 @@ class ManyVisitsWithMockLocationProvider extends Fixture MockLocationProvider::$locations = array( self::makeLocation('Toronto', 'ON', 'CA', $lat = null, $long = null, $isp = 'comcast.net'), - self::makeLocation('Nice', 'B8', 'FR', $lat = null, $long = null, $isp = 'comcast.net'), + self::makeLocation('Nice', 'PAC', 'FR', $lat = null, $long = null, $isp = 'comcast.net'), - self::makeLocation('Melbourne', '07', 'AU', $lat = null, $long = null, $isp = 'awesomeisp.com'), + self::makeLocation('Melbourne', 'VIC', 'AU', $lat = null, $long = null, $isp = 'awesomeisp.com'), - self::makeLocation('Yokohama', '19', 'JP'), + self::makeLocation('Yokohama', '14', 'JP'), ); } }
\ No newline at end of file diff --git a/tests/PHPUnit/Fixtures/SqlDump.php b/tests/PHPUnit/Fixtures/SqlDump.php index f5ebc7b364..8069c4b67e 100644 --- a/tests/PHPUnit/Fixtures/SqlDump.php +++ b/tests/PHPUnit/Fixtures/SqlDump.php @@ -85,8 +85,6 @@ class SqlDump extends Fixture } /** - * maybe this could use downloadAndUnzip(self::$geoLiteCityDbUrl, $geoIpOutputDir, 'GeoIPCity.dat'); - * * @param $dumpPath * @return int */ diff --git a/tests/PHPUnit/Fixtures/UITestFixture.php b/tests/PHPUnit/Fixtures/UITestFixture.php index 403abac8d5..b7a316f1f6 100644 --- a/tests/PHPUnit/Fixtures/UITestFixture.php +++ b/tests/PHPUnit/Fixtures/UITestFixture.php @@ -9,8 +9,6 @@ namespace Piwik\Tests\Fixtures; use Exception; use Piwik\API\Request; -use Piwik\AssetManager; -use Piwik\Access; use Piwik\Common; use Piwik\Date; use Piwik\Db; @@ -18,12 +16,12 @@ use Piwik\DbHelper; use Piwik\FrontController; use Piwik\Option; use Piwik\Piwik; +use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2; use Piwik\Plugins\PrivacyManager\IPAnonymizer; use Piwik\Plugins\SegmentEditor\API as APISegmentEditor; use Piwik\Plugins\UserCountry\LocationProvider; use Piwik\Plugins\UsersManager\API as UsersManagerAPI; use Piwik\Plugins\SitesManager\API as SitesManagerAPI; -use Piwik\Tests\Framework\Fixture; use Piwik\Plugins\VisitsSummary\API as VisitsSummaryAPI; use Piwik\Config as PiwikConfig; @@ -42,8 +40,6 @@ class UITestFixture extends SqlDump public function setUp() { - self::downloadGeoIpDbs(); - parent::setUp(); self::resetPluginsInstalledConfig(); @@ -57,7 +53,8 @@ class UITestFixture extends SqlDump ); // for proper geolocation - LocationProvider::setCurrentProvider(LocationProvider\GeoIp\Php::ID); + GeoIp2::$geoIPDatabaseDir = 'tests/lib/geoip-files'; + LocationProvider::setCurrentProvider(GeoIp2\Php::ID); IPAnonymizer::deactivate(); $this->addOverlayVisits(); |