Limit the fingerprint (#15886)

8 files changed, 314 insertions, 3 deletions

diff --git a/core/Config/IniFileChain.php b/core/Config/IniFileChain.php
index 17c7419b4f..e73ecd9818 100644
--- a/core/Config/IniFileChain.php
+++ b/core/Config/IniFileChain.php
@@ -97,6 +97,11 @@ class IniFileChain
      */
     public function set($name, $value)
     {
+        $name = $this->replaceSectionInvalidChars($name);
+        if ($value !== null) {
+            $value = $this->replaceInvalidChars($value);
+        }
+
         $this->mergedSettings[$name] = $value;
     }
 
@@ -538,4 +543,27 @@ class IniFileChain
         $writer = new IniWriter();
         return $writer->writeToString($values, $header);
     }
+
+    private function replaceInvalidChars($value)
+    {
+        if (is_array($value)) {
+            $result = [];
+            foreach ($value as $key => $arrayValue) {
+                $key = $this->replaceInvalidChars($key);
+                if (is_array($arrayValue)) {
+                    $arrayValue = $this->replaceInvalidChars($arrayValue);
+                }
+
+                $result[$key] = $arrayValue;
+            }
+            return $result;
+        } else {
+            return preg_replace('/[^a-zA-Z0-9_\[\]-]/', '', $value);
+        }
+    }
+
+    private function replaceSectionInvalidChars($value)
+    {
+        return preg_replace('/[^a-zA-Z0-9_-]/', '', $value);
+    }
 }
diff --git a/core/Tracker/FingerprintSalt.php b/core/Tracker/FingerprintSalt.php
new file mode 100644
index 0000000000..9896c074ca
--- /dev/null
+++ b/core/Tracker/FingerprintSalt.php
@@ -0,0 +1,88 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Tracker;
+
+use Piwik\Common;
+use Piwik\Date;
+use Piwik\Exception\InvalidRequestParameterException;
+use Piwik\Exception\UnexpectedWebsiteFoundException;
+use Piwik\Option;
+use Piwik\Piwik;
+use Piwik\SettingsServer;
+use Piwik\Site;
+use Piwik\Db as PiwikDb;
+
+class FingerprintSalt
+{
+    const OPTION_PREFIX = 'fingerprint_salt_';
+    const DELETE_FINGERPRINT_OLDER_THAN_SECONDS = 432000; // 5 days in seconds
+
+    public function generateSalt()
+    {
+        return Common::getRandomString(32);
+    }
+
+    public function deleteOldSalts()
+    {
+        // we want to make sure to delete salts that were created more than three days ago as they are likely not in
+        // use anymore. We should delete them to ensure the fingerprint is truly random for each day because if we used
+        // eg the regular salt then it would technically still be possible to try and regenerate the fingerprint based
+        // on certain information.
+        // Typically, only the salts for today and yesterday are used. However, if someone was to import historical data
+        // for the same day and this takes more than five days, then it could technically happen that we delete a
+        // fingerprint that is still in use now and as such after deletion a few visitors would have a new configId
+        // within one visit and such a new visit would be created. That should be very much edge case though.
+        $deleteSaltsCreatedBefore = Date::getNowTimestamp() - self::DELETE_FINGERPRINT_OLDER_THAN_SECONDS;
+        $options = Option::getLike(self::OPTION_PREFIX . '%');
+        $deleted = array();
+        foreach ($options as $name => $value) {
+            $value = $this->decode($value);
+            if (empty($value['time']) || $value['time'] < $deleteSaltsCreatedBefore) {
+                Option::delete($name);
+                $deleted[] = $name;
+            }
+        }
+
+        return $deleted;
+    }
+
+    public function getDateString(Date $date, $timezone)
+    {
+        $dateString = Date::factory($date->getTimestampUTC(), $timezone)->toString();
+        return $dateString;
+    }
+
+    private function encode($value)
+    {
+        return json_encode($value);
+    }
+
+    private function decode($value)
+    {
+        return @json_decode($value, true);
+    }
+
+    public function getSalt($dateString, $idSite)
+    {
+        $fingerprintSaltKey = self::OPTION_PREFIX . (int) $idSite . '_' . $dateString;
+        $salt = Option::get($fingerprintSaltKey);
+        if (!empty($salt)) {
+            $salt = $this->decode($salt);
+        }
+        if (empty($salt['value'])) {
+            $salt = array(
+                'value' => $this->generateSalt(),
+                'time' => Date::getNowTimestamp()
+            );
+            Option::set($fingerprintSaltKey, $this->encode($salt));
+        }
+        return $salt['value'];
+    }
+}
diff --git a/core/Tracker/Settings.php b/core/Tracker/Settings.php
index 84111c67a4..ee3eeb5774 100644
--- a/core/Tracker/Settings.php
+++ b/core/Tracker/Settings.php
@@ -10,6 +10,10 @@ namespace Piwik\Tracker;
 
 use Piwik\Config;
 use Piwik\Container\StaticContainer;
+use Piwik\Date;
+use Piwik\Option;
+use Piwik\SettingsServer;
+use Piwik\Site;
 use Piwik\Tracker;
 use Piwik\DeviceDetector\DeviceDetectorFactory;
 use Piwik\SettingsPiwik;
@@ -57,6 +61,31 @@ class Settings // TODO: merge w/ visitor recognizer or make it it's own service.
         }
 
         $browserLang = substr($request->getBrowserLanguage(), 0, 20); // limit the length of this string to match db
+        $trackerConfig = Config::getInstance()->Tracker;
+
+        $fingerprintSalt = '';
+
+        // fingerprint salt won't work when across multiple sites since all sites could have different timezones
+        // also cant add fingerprint salt for a specific day when we dont create new visit after midnight
+        if (!$this->isSameFingerprintsAcrossWebsites && !empty($trackerConfig['create_new_visit_after_midnight'])) {
+            $cache = Cache::getCacheWebsiteAttributes($request->getIdSite());
+            $date = Date::factory((int) $request->getCurrentTimestamp());
+            $fingerprintSaltKey = new FingerprintSalt();
+            $dateString = $fingerprintSaltKey->getDateString($date, $cache['timezone']);
+
+            if (!empty($cache[FingerprintSalt::OPTION_PREFIX . $dateString])) {
+                $fingerprintSalt = $cache[FingerprintSalt::OPTION_PREFIX . $dateString];
+            } else {
+                // we query the DB directly for requests older than 2-3 days...
+                $fingerprintSalt = $fingerprintSaltKey->getSalt($dateString, $request->getIdSite());
+            }
+
+            $fingerprintSalt .= $dateString;
+
+            if (defined('PIWIK_TEST_MODE') && PIWIK_TEST_MODE) {
+                $fingerprintSalt = ''; // use fixed value so they don't change randomly in tests
+            }
+        }
 
         return $this->getConfigHash(
             $request,
@@ -74,7 +103,8 @@ class Settings // TODO: merge w/ visitor recognizer or make it it's own service.
             $plugin_Silverlight,
             $plugin_Cookie,
             $ipAddress,
-            $browserLang);
+            $browserLang,
+            $fingerprintSalt);
     }
 
     /**
@@ -96,12 +126,13 @@ class Settings // TODO: merge w/ visitor recognizer or make it it's own service.
      * @param $plugin_Cookie
      * @param $ip
      * @param $browserLang
+     * @param $fingerprintHash
      * @return string
      */
     protected function getConfigHash(Request $request, $os, $browserName, $browserVersion, $plugin_Flash, $plugin_Java,
                                      $plugin_Director, $plugin_Quicktime, $plugin_RealPlayer, $plugin_PDF,
                                      $plugin_WindowsMedia, $plugin_Gears, $plugin_Silverlight, $plugin_Cookie, $ip,
-                                     $browserLang)
+                                     $browserLang, $fingerprintHash)
     {
         // prevent the config hash from being the same, across different Piwik instances
         // (limits ability of different Piwik instances to cross-match users)
@@ -114,7 +145,8 @@ class Settings // TODO: merge w/ visitor recognizer or make it it's own service.
             . $plugin_WindowsMedia . $plugin_Gears . $plugin_Silverlight . $plugin_Cookie
             . $ip
             . $browserLang
-            . $salt;
+            . $salt
+            . $fingerprintHash;
 
         if (!$this->isSameFingerprintsAcrossWebsites) {
             $configString .= $request->getIdSite();
diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php
index 97180241cc..24d06d1b17 100644
--- a/plugins/CoreAdminHome/Tasks.php
+++ b/plugins/CoreAdminHome/Tasks.php
@@ -32,6 +32,7 @@ use Piwik\Scheduler\Schedule\SpecificTime;
 use Piwik\Settings\Storage\Backend\MeasurableSettingsTable;
 use Piwik\Tracker\Failures;
 use Piwik\Site;
+use Piwik\Tracker\FingerprintSalt;
 use Piwik\Tracker\Visit\ReferrerSpamFilter;
 use Psr\Log\LoggerInterface;
 use Piwik\SettingsPiwik;
@@ -67,6 +68,8 @@ class Tasks extends \Piwik\Plugin\Tasks
         // sure all archives that need to be invalidated get invalidated
         $this->daily('invalidateOutdatedArchives', null, self::HIGH_PRIORITY);
 
+        $this->daily('deleteOldFingerprintSalts', null, self::HIGH_PRIORITY);
+
         // general data purge on older archive tables, executed daily
         $this->daily('purgeOutdatedArchives', null, self::HIGH_PRIORITY);
 
@@ -88,6 +91,11 @@ class Tasks extends \Piwik\Plugin\Tasks
         $this->scheduleTrackingCodeReminderChecks();
     }
 
+    public function deleteOldFingerprintSalts()
+    {
+        StaticContainer::get(FingerprintSalt::class)->deleteOldSalts();
+    }
+
     public function invalidateOutdatedArchives()
     {
         if (!Rules::isBrowserTriggerEnabled()) {
diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php
index 89c62252d5..b4e0c03d5a 100644
--- a/plugins/CoreAdminHome/tests/Integration/TasksTest.php
+++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php
@@ -133,6 +133,7 @@ class TasksTest extends IntegrationTestCase
 
         $expected = [
             'invalidateOutdatedArchives.',
+            'deleteOldFingerprintSalts.',
             'purgeOutdatedArchives.',
             'purgeInvalidatedArchives.',
             'purgeOrphanedArchives.',
diff --git a/plugins/SitesManager/SitesManager.php b/plugins/SitesManager/SitesManager.php
index 1320c0f6f2..2b8e767b26 100644
--- a/plugins/SitesManager/SitesManager.php
+++ b/plugins/SitesManager/SitesManager.php
@@ -13,12 +13,14 @@ use Piwik\API\Request;
 use Piwik\Common;
 use Piwik\Config;
 use Piwik\Container\StaticContainer;
+use Piwik\Date;
 use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Option;
 use Piwik\Piwik;
 use Piwik\Plugins\CoreHome\SystemSummary;
 use Piwik\Settings\Storage\Backend\MeasurableSettingsTable;
 use Piwik\Tracker\Cache;
+use Piwik\Tracker\FingerprintSalt;
 use Piwik\Tracker\Model as TrackerModel;
 use Piwik\Session\SessionNamespace;
 
@@ -197,6 +199,16 @@ class SitesManager extends \Piwik\Plugin
         $array['timezone'] = $this->getTimezoneFromWebsite($website);
         $array['ts_created'] = $website['ts_created'];
         $array['type'] = $website['type'];
+
+        // we make sure to have the fingerprint salts for the last 3 days incl tmrw in the cache so we don't need to
+        // query the DB directly for these days
+        $datesToGenerateSalt = array(Date::now()->addDay(1), Date::now(), Date::now()->subDay(1), Date::now()->subDay(2));
+
+        $fingerprintSaltKey = new FingerprintSalt();
+        foreach ($datesToGenerateSalt as $date) {
+            $dateString = $fingerprintSaltKey->getDateString($date, $array['timezone']);
+            $array[FingerprintSalt::OPTION_PREFIX . $dateString] = $fingerprintSaltKey->getSalt($dateString, $idSite);
+        }
     }
 
     public function setTrackerCacheGeneral(&$cache)
diff --git a/tests/PHPUnit/Integration/Tracker/FingerprintSaltTest.php b/tests/PHPUnit/Integration/Tracker/FingerprintSaltTest.php
new file mode 100644
index 0000000000..4680ca3d00
--- /dev/null
+++ b/tests/PHPUnit/Integration/Tracker/FingerprintSaltTest.php
@@ -0,0 +1,109 @@
+<?php
+/**
+ * Matomo - free/libre analytics platform
+ *
+ * @link https://matomo.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Tests\Integration\Tracker;
+
+use Piwik\Config;
+use Piwik\Date;
+use Piwik\Plugins\SitesManager\API;
+use Piwik\Tests\Framework\Mock\FakeAccess;
+use Piwik\Tracker\Action;
+use Piwik\Tracker\FingerprintSalt;
+use Piwik\Tracker\PageUrl;
+use Piwik\Tracker\Request;
+use Piwik\Translate;
+use Piwik\Plugin\Manager as PluginManager;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * @group Core
+ * @group ActionTest
+ */
+class FingerprintSaltTest extends IntegrationTestCase
+{
+    /**
+     * @var FingerprintSalt
+     */
+    private $fingerprintSalt;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        $this->fingerprintSalt = new FingerprintSalt();
+    }
+
+    public function test_generateSalt()
+    {
+        $salt = $this->fingerprintSalt->generateSalt();
+        $this->assertEquals(32, strlen($salt));
+        $this->assertTrue(ctype_alnum($salt));
+    }
+
+    public function test_generateSalt_isRandom()
+    {
+        $this->assertNotSame($this->fingerprintSalt->generateSalt(), $this->fingerprintSalt->generateSalt());
+    }
+
+    public function test_getDateString()
+    {
+        $date = Date::factory('2020-05-05 14:04:05');
+        $this->assertSame('2020-05-06',$this->fingerprintSalt->getDateString($date, 'Pacific/Auckland'));
+        $this->assertSame('2020-05-05',$this->fingerprintSalt->getDateString($date, 'Europe/Berlin'));
+    }
+
+    public function test_getSalt_remembersSaltPerSite()
+    {
+        $salt05_1 = $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 1);
+        $salt06_1 = $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 1);
+        $salt05_2 = $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 2);
+        $salt06_2 = $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 2);
+
+        $this->assertNotSame($salt05_1, $salt06_1);
+        $this->assertNotSame($salt05_2, $salt06_2);
+        $this->assertNotSame($salt06_1, $salt06_2);
+
+        $this->assertSame($salt05_1, $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 1));
+        $this->assertSame($salt06_1, $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 1));
+        $this->assertSame($salt05_2, $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 2));
+    }
+
+    public function test_deleteOldSalts_whenNothingToDelete()
+    {
+        $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 1);
+        $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 1);
+
+        Date::$now = time() - FingerprintSalt::DELETE_FINGERPRINT_OLDER_THAN_SECONDS + 30;// they would expire in 30 seconds
+        $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 2);
+        $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 2);
+
+        Date::$now = time();
+        $this->assertSame(array(), $this->fingerprintSalt->deleteOldSalts());
+    }
+
+    public function test_deleteOldSalts_someToBeDeleted()
+    {
+        $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 1);
+
+        Date::$now = time() - FingerprintSalt::DELETE_FINGERPRINT_OLDER_THAN_SECONDS - 30; // these entries should be expired
+        $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 1);
+        $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 2);
+        $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 2);
+
+        Date::$now = time();
+        $this->assertSame(array(
+            'fingerprint_salt_1_2020-05-06',
+            'fingerprint_salt_2_2020-05-05',
+            'fingerprint_salt_2_2020-05-06'
+        ), $this->fingerprintSalt->deleteOldSalts());
+
+        // executing it again wont delete anything
+        $this->assertSame(array(), $this->fingerprintSalt->deleteOldSalts());
+    }
+
+}
diff --git a/tests/PHPUnit/Unit/Config/IniFileChainTest.php b/tests/PHPUnit/Unit/Config/IniFileChainTest.php
index 2d6c56a6d6..f8e88ea058 100644
--- a/tests/PHPUnit/Unit/Config/IniFileChainTest.php
+++ b/tests/PHPUnit/Unit/Config/IniFileChainTest.php
@@ -8,6 +8,7 @@
 namespace Piwik\Tests\Unit\Config;
 
 use PHPUnit_Framework_TestCase;
+use Piwik\Config;
 use Piwik\Config\IniFileChain;
 
 /**
@@ -379,4 +380,36 @@ class IniFileChainTest extends PHPUnit_Framework_TestCase
         $actualOutput = $fileChain->dumpChanges($header);
         $this->assertEquals($expectedDumpChanges, $actualOutput);
     }
+
+
+    public function test_dump_handlesSpecialCharsCorrectly()
+    {
+        $config = new IniFileChain();
+        $config->set('first', ["a[]\n\n[d]\n\nb=4" => "\n\n[def]\na=b"]);
+        $config->set('second', ["a[]\n\n[d]b=4" => 'b']);
+        $config->set('thir][d]', ['a' => 'b']);
+        $config->set("four]\n\n[def]\n", ['d[]' => 'e']);
+        $out = $config->dump();
+
+        $expected = <<<END
+[first]
+a[][d]b4 = "
+
+[def]
+a=b"
+
+[second]
+a[][d]b4 = "b"
+
+[third]
+a = "b"
+
+[fourdef]
+d[] = "e"
+
+
+END;
+
+        $this->assertEquals($expected, $out);
+    }
 }
\ No newline at end of file