Allow using paid db-ip urls for auto update (#15606)

* Allow using paid db-ip urls for auto update

* apply review feedback

* Adds some Unit tests for new GeoIP2AutoUpdater methods

2 files changed, 185 insertions, 2 deletions

diff --git a/plugins/GeoIp2/GeoIP2AutoUpdater.php b/plugins/GeoIp2/GeoIP2AutoUpdater.php
index 4af0e8c4b2..423256be83 100644
--- a/plugins/GeoIp2/GeoIP2AutoUpdater.php
+++ b/plugins/GeoIp2/GeoIP2AutoUpdater.php
@@ -130,7 +130,10 @@ class GeoIP2AutoUpdater extends Task
         $logger = StaticContainer::get(LoggerInterface::class);
 
         $url = trim($url);
-        if ($this->isDbIpUrl($url)) {
+
+        if (self::isPaidDbIpUrl($url)) {
+            $url = $this->fetchPaidDbIpUrl($url);
+        } else if (self::isDbIpUrl($url)) {
             $url = $this->getDbIpUrlWithLatestDate($url);
         }
 
@@ -731,7 +734,7 @@ class GeoIP2AutoUpdater extends Task
         return LocationProviderGeoIp2::$dbNames[$dbType][0] . '.' . $ext;
     }
 
-    private function getDbIpUrlWithLatestDate($url)
+    protected function getDbIpUrlWithLatestDate($url)
     {
         $today = Date::today();
         return preg_replace('/-\d{4}-\d{2}\./', '-' . $today->toString('Y-m') . '.', $url);
@@ -741,4 +744,35 @@ class GeoIP2AutoUpdater extends Task
     {
         return !! preg_match('/db-ip\.com/', $url);
     }
+
+    protected static function isPaidDbIpUrl($url)
+    {
+        return !! preg_match('/db-ip\.com\/account\/[0-9a-z]+\/db/', $url);
+    }
+
+    protected function fetchPaidDbIpUrl($url)
+    {
+        $content = trim($this->fetchUrl($url));
+
+        if (0 === strpos($content, 'http')) {
+            return $content;
+        }
+
+        $content = json_decode($content, true);
+
+        if (!empty($content['mmdb']['url'])) {
+            return $content['mmdb']['url'];
+        }
+
+        if (!empty($content['url'])) {
+            return $content['url'];
+        }
+
+        throw new Exception('Unable to determine download url');
+    }
+
+    protected function fetchUrl($url)
+    {
+        return Http::fetchRemoteFile($url);
+    }
 }
diff --git a/plugins/GeoIp2/tests/Unit/GeoIP2AutoUpdaterTest.php b/plugins/GeoIp2/tests/Unit/GeoIP2AutoUpdaterTest.php
new file mode 100644
index 0000000000..ce68ee14af
--- /dev/null
+++ b/plugins/GeoIp2/tests/Unit/GeoIP2AutoUpdaterTest.php
@@ -0,0 +1,149 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\GeoIp2\tests\Unit;
+
+use Piwik\DataTable;
+use Piwik\DataTable\Row;
+use Piwik\Plugins\GeoIp2\GeoIP2AutoUpdater;
+
+class public_GeoIP2AutoUpdater extends GeoIP2AutoUpdater
+{
+    public static function isPaidDbIpUrl($url)
+    {
+        return parent::isPaidDbIpUrl($url);
+    }
+
+    public function fetchPaidDbIpUrl($url)
+    {
+        return parent::fetchPaidDbIpUrl($url);
+    }
+}
+
+/**
+ * @group GeoIP2AutoUpdater
+ */
+class GeoIP2AutoUpdaterTest extends \PHPUnit\Framework\TestCase
+{
+    /**
+     * @dataProvider getPaidDbTestUrls
+     */
+    public function testIsPaidDbIpUrl($expected, $url)
+    {
+        $this->assertEquals($expected, public_GeoIP2AutoUpdater::isPaidDbIpUrl($url));
+    }
+
+    public function getPaidDbTestUrls()
+    {
+        return [
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-country/'],
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-location/mmdb/'],
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-location-isp/mmdb/url'],
+            [false, 'https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb'],
+            [false, 'https://download.db-ip.com/free/dbip-country-lite-2020-02.mmdb.gz'],
+        ];
+    }
+
+    /**
+     * @dataProvider getDbTestUrls
+     */
+    public function testIsDbIpUrl($expected, $url)
+    {
+        $this->assertEquals($expected, GeoIP2AutoUpdater::isDbIpUrl($url));
+    }
+
+    public function getDbTestUrls()
+    {
+        return [
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-country/'],
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-location/mmdb/'],
+            [true, 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-location-isp/mmdb/url'],
+            [true, 'https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb'],
+            [true, 'https://download.db-ip.com/free/dbip-country-lite-2020-02.mmdb.gz'],
+            [false, 'https://dbip.com/free/dbip-country-lite-2020-02.mmdb.gz'],
+        ];
+    }
+
+    public function testFetchPaidUrlForPlainUrl()
+    {
+        $url = 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-country/mmdb/url';
+
+        $mock = $this->getMockBuilder(public_GeoIP2AutoUpdater::class)
+            ->setMethods(['fetchUrl'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $mock->expects($this->once())->method('fetchUrl')->with($url)->willReturn('https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb');
+
+        $determinedUrl = $mock->fetchPaidDbIpUrl($url);
+
+        $this->assertEquals('https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb', $determinedUrl);
+    }
+
+    public function testFetchPaidUrlForMmdbJson()
+    {
+        $url = 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-country/mmdb';
+
+        $mock = $this->getMockBuilder(public_GeoIP2AutoUpdater::class)
+            ->setMethods(['fetchUrl'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $mock->expects($this->once())->method('fetchUrl')->with($url)->willReturn('
+{
+    "url": "https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb",
+    "name": "dbip-country-2020-02-22.mmdb.gz",
+    "date": "February 22nd 2020",
+    "size": 8807222,
+    "rows": 808624,
+    "md5sum": "dd8250ca45ad42dd5c3a63670ff46968",
+    "sha1sum": "062615e15dd1496ac9fddc311231efa2d75f09d6"
+}');
+
+        $determinedUrl = $mock->fetchPaidDbIpUrl($url);
+
+        $this->assertEquals('https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb', $determinedUrl);
+    }
+
+    public function testFetchPaidUrlForFullJson()
+    {
+        $url = 'https://db-ip.com/account/ad446bf4cb9a44e4fff3f215deabc710f12f3/db/ip-to-country';
+
+        $mock = $this->getMockBuilder(public_GeoIP2AutoUpdater::class)
+            ->setMethods(['fetchUrl'])
+            ->disableOriginalConstructor()
+            ->getMock();
+
+        $mock->expects($this->once())->method('fetchUrl')->with($url)->willReturn('
+{
+    "csv": {
+        "url": "https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.csv",
+        "name": "dbip-country-2020-02-22.csv.gz",
+        "date": "February 22nd 2020",
+        "size": 35113592,
+        "rows": 808314,
+        "md5sum": "22cd9abcc07e6b5c1c0fe89eef4503e2",
+        "sha1sum": "d3cc6e7ed30cc58abcc77ae73318d63af2687b06",
+        "version": 3
+    },
+    "mmdb": {
+        "url": "https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb",
+        "name": "dbip-country-2020-02-22.mmdb.gz",
+        "date": "February 22nd 2020",
+        "size": 8807222,
+        "rows": 808314,
+        "md5sum": "dd8250ca45ad42c55abc63670ff46968",
+        "sha1sum": "062615e15e01496ac9fddabc1231efa2d75f09d6"
+    }
+}');
+
+        $determinedUrl = $mock->fetchPaidDbIpUrl($url);
+
+        $this->assertEquals('https://download.db-ip.com/key/ad446bf4cb9a44e4fff3f215deabc710f12f3.mmdb', $determinedUrl);
+    }
+}