Make it possible to automatically set up geoip2 in installation (#15939)

* make it possible to automatically set up geoip2 in installation

* reenable skipped installation ui tests

* updates expected UI files

* do download in hourly task

* trigger an async run of scheduled tasks to set up geoip2

* directly execute the task if possible and schedule the task only if not

* improve text

* updates UI test

13 files changed, 100 insertions, 22 deletions

diff --git a/plugins/GeoIp2/GeoIP2AutoUpdater.php b/plugins/GeoIp2/GeoIP2AutoUpdater.php
index c093bfcd6f..65903e94d7 100644
--- a/plugins/GeoIp2/GeoIP2AutoUpdater.php
+++ b/plugins/GeoIp2/GeoIP2AutoUpdater.php
@@ -15,13 +15,14 @@ use GeoIp2\Database\Reader;
 use Piwik\Common;
 use Piwik\Container\StaticContainer;
 use Piwik\Date;
-use Piwik\Filesystem;
 use Piwik\Http;
 use Piwik\Log;
 use Piwik\Option;
 use Piwik\Piwik;
 use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2 AS LocationProviderGeoIp2;
 use Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2\Php;
+use Piwik\Plugins\UserCountry\LocationProvider;
+use Piwik\Scheduler\Schedule\Hourly;
 use Piwik\Scheduler\Scheduler;
 use Piwik\Scheduler\Task;
 use Piwik\Scheduler\Timetable;
@@ -47,6 +48,8 @@ class GeoIP2AutoUpdater extends Task
 
     const LAST_RUN_TIME_OPTION_NAME = 'geoip2.updater_last_run_time';
 
+    const AUTO_SETUP_OPTION_NAME = 'geoip2.autosetup';
+
     private static $urlOptions = array(
         'loc' => self::LOC_URL_OPTION_NAME,
         'isp' => self::ISP_URL_OPTION_NAME,
@@ -82,6 +85,10 @@ class GeoIP2AutoUpdater extends Task
                 break;
         }
 
+        if (Option::get(self::AUTO_SETUP_OPTION_NAME)) {
+            $schedulePeriod = new Hourly();
+        }
+
         parent::__construct($this, 'update', null, $schedulePeriod, Task::LOWEST_PRIORITY);
     }
 
@@ -111,6 +118,15 @@ class GeoIP2AutoUpdater extends Task
         }
 
         $this->performRedundantDbChecks();
+
+        if (Option::get(self::AUTO_SETUP_OPTION_NAME)) {
+            Option::delete(self::AUTO_SETUP_OPTION_NAME);
+            LocationProvider::setCurrentProvider(Php::ID);
+            /** @var Scheduler $scheduler */
+            $scheduler = StaticContainer::getContainer()->get('Piwik\Scheduler\Scheduler');
+            // reschedule to ensure it's not run again in an hour
+            $scheduler->rescheduleTask(new GeoIP2AutoUpdater());
+        }
     }
 
     /**
@@ -262,7 +278,8 @@ class GeoIP2AutoUpdater extends Task
                 $dbFilename = $php->detectDatabaseType($dbType) . '.mmdb';
             }
         } else {
-            $ext = end(explode(basename($path), '.', 2));
+            $parts = explode(basename($path), '.', 2);
+            $ext = end($parts);
             throw new Exception(Piwik::translate('GeoIp2_UnsupportedArchiveType', "'$ext'"));
         }
 
diff --git a/plugins/GeoIp2/GeoIp2.php b/plugins/GeoIp2/GeoIp2.php
index ffba09bd98..2f1c2e6748 100644
--- a/plugins/GeoIp2/GeoIp2.php
+++ b/plugins/GeoIp2/GeoIp2.php
@@ -8,7 +8,13 @@
  */
 namespace Piwik\Plugins\GeoIp2;
 
+use Piwik\CliMulti;
+use Piwik\Container\StaticContainer;
+use Piwik\Option;
+use Piwik\Piwik;
+use Piwik\Plugins\Installation\FormDefaultSettings;
 use Piwik\Plugins\UserCountry\LocationProvider;
+use Piwik\Scheduler\Scheduler;
 
 /**
  *
@@ -18,8 +24,10 @@ class GeoIp2 extends \Piwik\Plugin
     public function registerEvents()
     {
         return array(
-            'AssetManager.getJavaScriptFiles'        => 'getJsFiles',
-            'Translate.getClientSideTranslationKeys' => 'getClientSideTranslationKeys',
+            'AssetManager.getJavaScriptFiles'         => 'getJsFiles',
+            'Translate.getClientSideTranslationKeys'  => 'getClientSideTranslationKeys',
+            'Installation.defaultSettingsForm.init'   => 'installationFormInit',
+            'Installation.defaultSettingsForm.submit' => 'installationFormSubmit',
         );
     }
 
@@ -50,4 +58,56 @@ class GeoIp2 extends \Piwik\Plugin
         $translationKeys[] = "General_Save";
         $translationKeys[] = "General_Continue";
     }
+
+    /**
+     * Customize the Installation "default settings" form.
+     *
+     * @param FormDefaultSettings $form
+     */
+    public function installationFormInit(FormDefaultSettings $form)
+    {
+        $form->addElement('checkbox', 'setup_geoip2', null,
+            [
+                'content' => '<div class="form-help">' . Piwik::translate('GeoIp2_AutomaticSetupDescription', ['<a rel="noreferrer noopener" href="https://db-ip.com/?refid=mtm">','</a>']) . '</div> &nbsp;&nbsp;' . Piwik::translate('GeoIp2_AutomaticSetup')
+            ]
+        );
+
+        // default values
+        $form->addDataSource(new \HTML_QuickForm2_DataSource_Array([
+            'setup_geoip2' => true,
+        ]));
+    }
+
+    /**
+     * Process the submit on the Installation "default settings" form.
+     *
+     * @param FormDefaultSettings $form
+     */
+    public function installationFormSubmit(FormDefaultSettings $form)
+    {
+        $setupGeoIp2 = (bool) $form->getSubmitValue('setup_geoip2');
+
+        if ($setupGeoIp2) {
+            Option::set(GeoIP2AutoUpdater::AUTO_SETUP_OPTION_NAME, true);
+            GeoIP2AutoUpdater::setUpdaterOptions([
+                'loc' => \Piwik\Plugins\GeoIp2\LocationProvider\GeoIp2::getDbIpLiteUrl(),
+                'period' => GeoIP2AutoUpdater::SCHEDULE_PERIOD_MONTHLY
+            ]);
+
+            $cliMulti = new CliMulti();
+
+            // directly trigger the update task if possible
+            // otherwise ensure it will be run soonish as scheduled task
+            if ($cliMulti->supportsAsync()) {
+                $phpCli = new CliMulti\CliPhp();
+                $command = sprintf('%s %s/console core:run-scheduled-tasks --force "Piwik\Plugins\GeoIp2\GeoIP2AutoUpdater.update" > /dev/null 2>&1 &',
+                    $phpCli->findPhpBinary(), PIWIK_INCLUDE_PATH);
+                shell_exec($command);
+            } else {
+                /** @var Scheduler $scheduler */
+                $scheduler = StaticContainer::getContainer()->get('Piwik\Scheduler\Scheduler');
+                $scheduler->rescheduleTask(new GeoIP2AutoUpdater());
+            }
+        }
+    }
 }
diff --git a/plugins/GeoIp2/lang/en.json b/plugins/GeoIp2/lang/en.json
index 158028266e..bee4e21e16 100644
--- a/plugins/GeoIp2/lang/en.json
+++ b/plugins/GeoIp2/lang/en.json
@@ -1,6 +1,8 @@
 {
   "GeoIp2": {
     "AssumingNonApache": "Cannot find apache_get_modules function, assuming non-Apache webserver.",
+    "AutomaticSetup": "Automatically configure geolocation using a dbip database",
+    "AutomaticSetupDescription": "For a proper geolocation Matomo requires an external database. Using this option, Matomo will automatically be configured to download and use the latest dbip city level database. [%1$sView licensing terms%2$s]",
     "CannotFindGeoIPDatabaseInArchive": "No valid DBIP / GeoIP database could be found in tar archive %1$s!",
     "CannotFindGeoIPServerVar": "The %s variable is not set. Your server may not be configured correctly.",
     "CannotListContent": "Couldn't list content for %1$s: %2$s",
diff --git a/plugins/Installation/Controller.php b/plugins/Installation/Controller.php
index 7c7d34cd7f..b7dbd5a60a 100644
--- a/plugins/Installation/Controller.php
+++ b/plugins/Installation/Controller.php
@@ -27,7 +27,6 @@ use Piwik\Plugins\SitesManager\API as APISitesManager;
 use Piwik\Plugins\UsersManager\API as APIUsersManager;
 use Piwik\Plugins\UsersManager\NewsletterSignup;
 use Piwik\Plugins\UsersManager\UserUpdater;
-
 use Piwik\ProxyHeaders;
 use Piwik\SettingsPiwik;
 use Piwik\Tracker\TrackerCodeGenerator;
diff --git a/plugins/Installation/tests/UI/Installation_spec.js b/plugins/Installation/tests/UI/Installation_spec.js
index 4f667bc69a..058275bc76 100644
--- a/plugins/Installation/tests/UI/Installation_spec.js
+++ b/plugins/Installation/tests/UI/Installation_spec.js
@@ -88,7 +88,7 @@ describe("Installation", function () {
 
         expect(await page.screenshot({ fullPage: true })).to.matchImage('db_setup_fail');
     });
-return; // next test times out
+
     it("should display the tables created page when next is clicked on the db setup page w/ correct info entered in the form", async function() {
         const dbInfo = testEnvironment.readDbInfoFromConfig();
         const username = dbInfo.username;
diff --git a/plugins/Installation/tests/UI/expected-screenshots/Installation_congrats.png b/plugins/Installation/tests/UI/expected-screenshots/Installation_congrats.png
index 84dc6f6ee5..20ac43f39e 100644
--- a/plugins/Installation/tests/UI/expected-screenshots/Installation_congrats.png
+++ b/plugins/Installation/tests/UI/expected-screenshots/Installation_congrats.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9cebcf1acea004e2e2cf7b52fa53a4a4fae19d052ebaae843f37a3700dfb7be1
-size 170481
+oid sha256:92bf44ce1f73faa4c47f15f2a93a474d31977bae3e316bdf93d4037d259d571a
+size 197253
diff --git a/plugins/Installation/tests/UI/expected-screenshots/Installation_db_created.png b/plugins/Installation/tests/UI/expected-screenshots/Installation_db_created.png
index b4b7c4da10..65a2015fce 100644
--- a/plugins/Installation/tests/UI/expected-screenshots/Installation_db_created.png
+++ b/plugins/Installation/tests/UI/expected-screenshots/Installation_db_created.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a92c8bffb687c7ccfa1f6428dc2b601d88c291a8c460f6e6fcc49726f16112ac
-size 45398
+oid sha256:b17e4b346e715a0f59c2105cbe912606f76f3ee57ed902ea10fe47d25543c5e7
+size 44592
diff --git a/plugins/Installation/tests/UI/expected-screenshots/Installation_js_tracking.png b/plugins/Installation/tests/UI/expected-screenshots/Installation_js_tracking.png
index 92a2693c4f..5a2467f16a 100644
--- a/plugins/Installation/tests/UI/expected-screenshots/Installation_js_tracking.png
+++ b/plugins/Installation/tests/UI/expected-screenshots/Installation_js_tracking.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:741792b60f6490bd78a8950342b550d66ddc856f76fc7aa704c2e8a1a8cc8634
-size 225156
+oid sha256:36333c400e55951cfb66bb79d4aa029799927c655b0af1ed46d6b90edfbdd161
+size 239861
diff --git a/plugins/Installation/tests/UI/expected-screenshots/Installation_setup_website.png b/plugins/Installation/tests/UI/expected-screenshots/Installation_setup_website.png
index 96bd2dd071..ae50c0bb5d 100644
--- a/plugins/Installation/tests/UI/expected-screenshots/Installation_setup_website.png
+++ b/plugins/Installation/tests/UI/expected-screenshots/Installation_setup_website.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d64d827204c0700f762091a433a7d81378656aa5bd59f28f02e2925a4a8eaf5c
-size 76064
+oid sha256:05723d3b9fd0964b531d8521d82f38ddfae9e7858161965d4fdb3ab1c6822a4a
+size 77528
diff --git a/plugins/Installation/tests/UI/expected-screenshots/Installation_setup_website_fail.png b/plugins/Installation/tests/UI/expected-screenshots/Installation_setup_website_fail.png
index 056554c1a3..d4b304ed10 100644
--- a/plugins/Installation/tests/UI/expected-screenshots/Installation_setup_website_fail.png
+++ b/plugins/Installation/tests/UI/expected-screenshots/Installation_setup_website_fail.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:497ecc743c1ee3fc6be4a0a8705209cbf5449f726f5f14dbd5c8b9ef0d6b764b
-size 82042
+oid sha256:289d3963995836f34f28a8d7267db5cf7cc7d7424b08bbd6da35625adf5a2156
+size 82430
diff --git a/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser.png b/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser.png
index 9c48931855..c4e083a3ef 100644
--- a/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser.png
+++ b/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f3f23ecda363335121e9084c23f48215d9f461a7263d48bec1ff93bdca990cf9
-size 90983
+oid sha256:861b47bd63e385dc821c17ea0d03b55fb2ccffb859f197af5394cf7e6ac7746b
+size 89603
diff --git a/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser_de.png b/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser_de.png
index 8a2bc28bee..78cbbebdd5 100644
--- a/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser_de.png
+++ b/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser_de.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fd8bcf6b6f519cb339a4e4f7e1a3235e3afadc34ae23dd3c584a58b2eb199b40
-size 106822
+oid sha256:6e35a01b13411e38a785b341f70eab32577b3b9140653f7877b2bfd65b3cdc8e
+size 106065
diff --git a/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser_fail.png b/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser_fail.png
index fd8e902827..172f5c8e97 100644
--- a/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser_fail.png
+++ b/plugins/Installation/tests/UI/expected-screenshots/Installation_superuser_fail.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ba81a4fba3604dfde7495bf25769d89b2430bd7c518e923558db011542b76831
-size 105855
+oid sha256:a705b0d1d45e8477d9c4c055584736717e0aba329b86d77909b5e0a58865813f
+size 103334