Merge branch '3.x-dev' into 4.x-dev

4 files changed, 231 insertions, 3 deletions

diff --git a/plugins/GeoIp2/GeoIP2AutoUpdater.php b/plugins/GeoIp2/GeoIP2AutoUpdater.php
index 4af0e8c4b2..423256be83 100644
--- a/plugins/GeoIp2/GeoIP2AutoUpdater.php
+++ b/plugins/GeoIp2/GeoIP2AutoUpdater.php
@@ -130,7 +130,10 @@ class GeoIP2AutoUpdater extends Task
         $logger = StaticContainer::get(LoggerInterface::class);
 
         $url = trim($url);
-        if ($this->isDbIpUrl($url)) {
+
+        if (self::isPaidDbIpUrl($url)) {
+            $url = $this->fetchPaidDbIpUrl($url);
+        } else if (self::isDbIpUrl($url)) {
             $url = $this->getDbIpUrlWithLatestDate($url);
         }
 
@@ -731,7 +734,7 @@ class GeoIP2AutoUpdater extends Task
         return LocationProviderGeoIp2::$dbNames[$dbType][0] . '.' . $ext;
     }
 
-    private function getDbIpUrlWithLatestDate($url)
+    protected function getDbIpUrlWithLatestDate($url)
     {
         $today = Date::today();
         return preg_replace('/-\d{4}-\d{2}\./', '-' . $today->toString('Y-m') . '.', $url);
@@ -741,4 +744,35 @@ class GeoIP2AutoUpdater extends Task
     {
         return !! preg_match('/db-ip\.com/', $url);
     }
+
+    protected static function isPaidDbIpUrl($url)
+    {
+        return !! preg_match('/db-ip\.com\/account\/[0-9a-z]+\/db/', $url);
+    }
+
+    protected function fetchPaidDbIpUrl($url)
+    {
+        $content = trim($this->fetchUrl($url));
+
+        if (0 === strpos($content, 'http')) {
+            return $content;
+        }
+
+        $content = json_decode($content, true);
+
+        if (!empty($content['mmdb']['url'])) {
+            return $content['mmdb']['url'];
+        }
+
+        if (!empty($content['url'])) {
+            return $content['url'];
+        }
+
+        throw new Exception('Unable to determine download url');
+    }
+
+    protected function fetchUrl($url)
+    {
+        return Http::fetchRemoteFile($url);
+    }
 }
diff --git a/plugins/GeoIp2/LocationProvider/GeoIp2/Php.php b/plugins/GeoIp2/LocationProvider/GeoIp2/Php.php
index f2c145e35e..310a273dc4 100644
--- a/plugins/GeoIp2/LocationProvider/GeoIp2/Php.php
+++ b/plugins/GeoIp2/LocationProvider/GeoIp2/Php.php
@@ -11,6 +11,7 @@ namespace Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2;
 use GeoIp2\Database\Reader;
 use GeoIp2\Exception\AddressNotFoundException;
 use MaxMind\Db\Reader\InvalidDatabaseException;
+use Piwik\Common;
 use Piwik\Log;
 use Piwik\Piwik;
 use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2;
@@ -238,11 +239,35 @@ class Php extends GeoIp2
         if (is_array($lookupResult->subdivisions) && count($lookupResult->subdivisions) > 0) {
             $subdivisions = $lookupResult->subdivisions;
             $subdivision = $this->determinSubdivision($subdivisions, $result[self::COUNTRY_CODE_KEY]);
-            $result[self::REGION_CODE_KEY] = strtoupper($subdivision->isoCode);
+            $result[self::REGION_CODE_KEY] = strtoupper($subdivision->isoCode) ?: $this->determineRegionIsoCodeByNameAndCountryCode($subdivision->name, $result[self::COUNTRY_CODE_KEY]);
             $result[self::REGION_NAME_KEY] = $subdivision->name;
         }
     }
 
+    /**
+     * Try to determine the ISO region code based on the region name and country code
+     *
+     * @param string $regionName
+     * @param string $countryCode
+     * @return string
+     */
+    protected function determineRegionIsoCodeByNameAndCountryCode($regionName, $countryCode)
+    {
+        $regionNames = self::getRegionNames();
+
+        if (empty($regionNames[$countryCode])) {
+            return '';
+        }
+
+        foreach ($regionNames[$countryCode] as $isoCode => $name) {
+            if (Common::mb_strtolower($name) === Common::mb_strtolower($regionName)) {
+                return $isoCode;
+            }
+        }
+
+        return '';
+    }
+
     protected function determinSubdivision($subdivisions, $countryCode)
     {
         if (in_array($countryCode, ['GB'])) {
diff --git a/plugins/GeoIp2/tests/Integration/LocationProviderTest.php b/plugins/GeoIp2/tests/Integration/LocationProviderTest.php
index bb681f7704..86d9ae5529 100644
--- a/plugins/GeoIp2/tests/Integration/LocationProviderTest.php
+++ b/plugins/GeoIp2/tests/Integration/LocationProviderTest.php
@@ -34,6 +34,26 @@ class LocationProviderTest extends \PHPUnit\Framework\TestCase
         ], $result);
     }
 
+    public function testGeoIP2CityWithoutRegionIsoCode()
+    {
+        // The IP 99.99.99.99 will only return a region name, based on that the region code should be determined
+        $locationProvider = new GeoIp2\Php(['loc' => ['GeoIP2-City.mmdb'], 'isp' => []]);
+        $result = $locationProvider->getLocation(['ip' => '99.99.99.99']);
+
+        $this->assertEquals([
+            'continent_name' => 'North America',
+            'continent_code' => 'NA',
+            'country_code' => 'US',
+            'country_name' => 'United States',
+            'city_name' => 'Englewood Cliffs',
+            'lat' => 40.892,
+            'long' => -73.947,
+            'postal_code' => null,
+            'region_code' => 'NJ',
+            'region_name' => 'New Jersey',
+        ], $result);
+    }
+
     public function testGeoIP2Country()
     {
         $locationProvider = new GeoIp2\Php(['loc' => ['GeoIP2-Country.mmdb'], 'isp' => []]);
diff --git a/plugins/GeoIp2/tests/Unit/GeoIP2AutoUpdaterTest.php b/plugins/GeoIp2/tests/Unit/GeoIP2AutoUpdaterTest.php
new file mode 100644
index 0000000000..ce68ee14af
--- /dev/null
+++ b/plugins/GeoIp2/tests/Unit/GeoIP2AutoUpdaterTest.php
@@ -0,0 +1,149 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\GeoIp2\tests\Unit;
+
+use Piwik\DataTable;
+use Piwik\DataTable\Row;
+use Piwik\Plugins\GeoIp2\GeoIP2AutoUpdater;
+
+class public_GeoIP2AutoUpdater extends GeoIP2AutoUpdater
+{
+    public static function isPaidDbIpUrl($url)
+    {
+        return parent::isPaidDbIpUrl($url);
+    }
+
+    public function fetchPaidDbIpUrl($url)
+    {
+        return parent::fetchPaidDbIpUrl($url);
+    }
+}
+
+/**
+ * @group GeoIP2AutoUpdater
+ */
+class GeoIP2AutoUpdaterTest extends \PHPUnit\Framework\TestCase
+{
+    /**
+     * @dataProvider getPaidDbTestUrls
+     */
+    public function testIsPaidDbIpUrl($expected, $url)
+    {
+        $this->assertEquals($expected, public_GeoIP2AutoUpdater::isPaidDbIpUrl($url));
+    }
+
+    public function getPaidDbTestUrls()
+    {
+        return [
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-country/'],
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-location/mmdb/'],
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-location-isp/mmdb/url'],
+            [false, 'https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb'],
+            [false, 'https://download.db-ip.com/free/dbip-country-lite-2020-02.mmdb.gz'],
+        ];
+    }
+
+    /**
+     * @dataProvider getDbTestUrls
+     */
+    public function testIsDbIpUrl($expected, $url)
+    {
+        $this->assertEquals($expected, GeoIP2AutoUpdater::isDbIpUrl($url));
+    }
+
+    public function getDbTestUrls()
+    {
+        return [
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-country/'],
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-location/mmdb/'],
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-location-isp/mmdb/url'],
+            [true, 'https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb'],
+            [true, 'https://download.db-ip.com/free/dbip-country-lite-2020-02.mmdb.gz'],
+            [false, 'https://dbip.com/free/dbip-country-lite-2020-02.mmdb.gz'],
+        ];
+    }
+
+    public function testFetchPaidUrlForPlainUrl()
+    {
+        $url = 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-country/mmdb/url';
+
+        $mock = $this->getMockBuilder(public_GeoIP2AutoUpdater::class)
+            ->setMethods(['fetchUrl'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $mock->expects($this->once())->method('fetchUrl')->with($url)->willReturn('https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb');
+
+        $determinedUrl = $mock->fetchPaidDbIpUrl($url);
+
+        $this->assertEquals('https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb', $determinedUrl);
+    }
+
+    public function testFetchPaidUrlForMmdbJson()
+    {
+        $url = 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-country/mmdb';
+
+        $mock = $this->getMockBuilder(public_GeoIP2AutoUpdater::class)
+            ->setMethods(['fetchUrl'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $mock->expects($this->once())->method('fetchUrl')->with($url)->willReturn('
+{
+    "url": "https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb",
+    "name": "dbip-country-2020-02-22.mmdb.gz",
+    "date": "February 22nd 2020",
+    "size": 8807222,
+    "rows": 808624,
+    "md5sum": "dd8250ca45ad42dd5c3a63670ff46968",
+    "sha1sum": "062615e15dd1496ac9fddc311231efa2d75f09d6"
+}');
+
+        $determinedUrl = $mock->fetchPaidDbIpUrl($url);
+
+        $this->assertEquals('https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb', $determinedUrl);
+    }
+
+    public function testFetchPaidUrlForFullJson()
+    {
+        $url = 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-country';
+
+        $mock = $this->getMockBuilder(public_GeoIP2AutoUpdater::class)
+            ->setMethods(['fetchUrl'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $mock->expects($this->once())->method('fetchUrl')->with($url)->willReturn('
+{
+    "csv": {
+        "url": "https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.csv",
+        "name": "dbip-country-2020-02-22.csv.gz",
+        "date": "February 22nd 2020",
+        "size": 35113592,
+        "rows": 808314,
+        "md5sum": "22cd9abcc07e6b5c1c0fe89eef4503e2",
+        "sha1sum": "d3cc6e7ed30cc58abcc77ae73318d63af2687b06",
+        "version": 3
+    },
+    "mmdb": {
+        "url": "https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb",
+        "name": "dbip-country-2020-02-22.mmdb.gz",
+        "date": "February 22nd 2020",
+        "size": 8807222,
+        "rows": 808314,
+        "md5sum": "dd8250ca45ad42c55abc63670ff46968",
+        "sha1sum": "062615e15e01496ac9fddabc1231efa2d75f09d6"
+    }
+}');
+
+        $determinedUrl = $mock->fetchPaidDbIpUrl($url);
+
+        $this->assertEquals('https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb', $determinedUrl);
+    }
+}