diff options
-rw-r--r-- | core/Config/IniFileChain.php | 28 | ||||
-rw-r--r-- | core/Tracker/FingerprintSalt.php | 88 | ||||
-rw-r--r-- | core/Tracker/Settings.php | 38 | ||||
-rw-r--r-- | plugins/CoreAdminHome/Tasks.php | 8 | ||||
-rw-r--r-- | plugins/CoreAdminHome/tests/Integration/TasksTest.php | 1 | ||||
-rw-r--r-- | plugins/SitesManager/SitesManager.php | 12 | ||||
-rw-r--r-- | tests/PHPUnit/Integration/Tracker/FingerprintSaltTest.php | 109 | ||||
-rw-r--r-- | tests/PHPUnit/Unit/Config/IniFileChainTest.php | 33 |
8 files changed, 314 insertions, 3 deletions
diff --git a/core/Config/IniFileChain.php b/core/Config/IniFileChain.php index 17c7419b4f..e73ecd9818 100644 --- a/core/Config/IniFileChain.php +++ b/core/Config/IniFileChain.php @@ -97,6 +97,11 @@ class IniFileChain */ public function set($name, $value) { + $name = $this->replaceSectionInvalidChars($name); + if ($value !== null) { + $value = $this->replaceInvalidChars($value); + } + $this->mergedSettings[$name] = $value; } @@ -538,4 +543,27 @@ class IniFileChain $writer = new IniWriter(); return $writer->writeToString($values, $header); } + + private function replaceInvalidChars($value) + { + if (is_array($value)) { + $result = []; + foreach ($value as $key => $arrayValue) { + $key = $this->replaceInvalidChars($key); + if (is_array($arrayValue)) { + $arrayValue = $this->replaceInvalidChars($arrayValue); + } + + $result[$key] = $arrayValue; + } + return $result; + } else { + return preg_replace('/[^a-zA-Z0-9_\[\]-]/', '', $value); + } + } + + private function replaceSectionInvalidChars($value) + { + return preg_replace('/[^a-zA-Z0-9_-]/', '', $value); + } } diff --git a/core/Tracker/FingerprintSalt.php b/core/Tracker/FingerprintSalt.php new file mode 100644 index 0000000000..9896c074ca --- /dev/null +++ b/core/Tracker/FingerprintSalt.php @@ -0,0 +1,88 @@ +<?php +/** + * Matomo - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Tracker; + +use Piwik\Common; +use Piwik\Date; +use Piwik\Exception\InvalidRequestParameterException; +use Piwik\Exception\UnexpectedWebsiteFoundException; +use Piwik\Option; +use Piwik\Piwik; +use Piwik\SettingsServer; +use Piwik\Site; +use Piwik\Db as PiwikDb; + +class FingerprintSalt +{ + const OPTION_PREFIX = 'fingerprint_salt_'; + const DELETE_FINGERPRINT_OLDER_THAN_SECONDS = 432000; // 5 days in seconds + + public function generateSalt() + { + return Common::getRandomString(32); + } + + public function deleteOldSalts() + { + // we want to make sure to delete salts that were created more than three days ago as they are likely not in + // use anymore. We should delete them to ensure the fingerprint is truly random for each day because if we used + // eg the regular salt then it would technically still be possible to try and regenerate the fingerprint based + // on certain information. + // Typically, only the salts for today and yesterday are used. However, if someone was to import historical data + // for the same day and this takes more than five days, then it could technically happen that we delete a + // fingerprint that is still in use now and as such after deletion a few visitors would have a new configId + // within one visit and such a new visit would be created. That should be very much edge case though. + $deleteSaltsCreatedBefore = Date::getNowTimestamp() - self::DELETE_FINGERPRINT_OLDER_THAN_SECONDS; + $options = Option::getLike(self::OPTION_PREFIX . '%'); + $deleted = array(); + foreach ($options as $name => $value) { + $value = $this->decode($value); + if (empty($value['time']) || $value['time'] < $deleteSaltsCreatedBefore) { + Option::delete($name); + $deleted[] = $name; + } + } + + return $deleted; + } + + public function getDateString(Date $date, $timezone) + { + $dateString = Date::factory($date->getTimestampUTC(), $timezone)->toString(); + return $dateString; + } + + private function encode($value) + { + return json_encode($value); + } + + private function decode($value) + { + return @json_decode($value, true); + } + + public function getSalt($dateString, $idSite) + { + $fingerprintSaltKey = self::OPTION_PREFIX . (int) $idSite . '_' . $dateString; + $salt = Option::get($fingerprintSaltKey); + if (!empty($salt)) { + $salt = $this->decode($salt); + } + if (empty($salt['value'])) { + $salt = array( + 'value' => $this->generateSalt(), + 'time' => Date::getNowTimestamp() + ); + Option::set($fingerprintSaltKey, $this->encode($salt)); + } + return $salt['value']; + } +} diff --git a/core/Tracker/Settings.php b/core/Tracker/Settings.php index 84111c67a4..ee3eeb5774 100644 --- a/core/Tracker/Settings.php +++ b/core/Tracker/Settings.php @@ -10,6 +10,10 @@ namespace Piwik\Tracker; use Piwik\Config; use Piwik\Container\StaticContainer; +use Piwik\Date; +use Piwik\Option; +use Piwik\SettingsServer; +use Piwik\Site; use Piwik\Tracker; use Piwik\DeviceDetector\DeviceDetectorFactory; use Piwik\SettingsPiwik; @@ -57,6 +61,31 @@ class Settings // TODO: merge w/ visitor recognizer or make it it's own service. } $browserLang = substr($request->getBrowserLanguage(), 0, 20); // limit the length of this string to match db + $trackerConfig = Config::getInstance()->Tracker; + + $fingerprintSalt = ''; + + // fingerprint salt won't work when across multiple sites since all sites could have different timezones + // also cant add fingerprint salt for a specific day when we dont create new visit after midnight + if (!$this->isSameFingerprintsAcrossWebsites && !empty($trackerConfig['create_new_visit_after_midnight'])) { + $cache = Cache::getCacheWebsiteAttributes($request->getIdSite()); + $date = Date::factory((int) $request->getCurrentTimestamp()); + $fingerprintSaltKey = new FingerprintSalt(); + $dateString = $fingerprintSaltKey->getDateString($date, $cache['timezone']); + + if (!empty($cache[FingerprintSalt::OPTION_PREFIX . $dateString])) { + $fingerprintSalt = $cache[FingerprintSalt::OPTION_PREFIX . $dateString]; + } else { + // we query the DB directly for requests older than 2-3 days... + $fingerprintSalt = $fingerprintSaltKey->getSalt($dateString, $request->getIdSite()); + } + + $fingerprintSalt .= $dateString; + + if (defined('PIWIK_TEST_MODE') && PIWIK_TEST_MODE) { + $fingerprintSalt = ''; // use fixed value so they don't change randomly in tests + } + } return $this->getConfigHash( $request, @@ -74,7 +103,8 @@ class Settings // TODO: merge w/ visitor recognizer or make it it's own service. $plugin_Silverlight, $plugin_Cookie, $ipAddress, - $browserLang); + $browserLang, + $fingerprintSalt); } /** @@ -96,12 +126,13 @@ class Settings // TODO: merge w/ visitor recognizer or make it it's own service. * @param $plugin_Cookie * @param $ip * @param $browserLang + * @param $fingerprintHash * @return string */ protected function getConfigHash(Request $request, $os, $browserName, $browserVersion, $plugin_Flash, $plugin_Java, $plugin_Director, $plugin_Quicktime, $plugin_RealPlayer, $plugin_PDF, $plugin_WindowsMedia, $plugin_Gears, $plugin_Silverlight, $plugin_Cookie, $ip, - $browserLang) + $browserLang, $fingerprintHash) { // prevent the config hash from being the same, across different Piwik instances // (limits ability of different Piwik instances to cross-match users) @@ -114,7 +145,8 @@ class Settings // TODO: merge w/ visitor recognizer or make it it's own service. . $plugin_WindowsMedia . $plugin_Gears . $plugin_Silverlight . $plugin_Cookie . $ip . $browserLang - . $salt; + . $salt + . $fingerprintHash; if (!$this->isSameFingerprintsAcrossWebsites) { $configString .= $request->getIdSite(); diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index 97180241cc..24d06d1b17 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -32,6 +32,7 @@ use Piwik\Scheduler\Schedule\SpecificTime; use Piwik\Settings\Storage\Backend\MeasurableSettingsTable; use Piwik\Tracker\Failures; use Piwik\Site; +use Piwik\Tracker\FingerprintSalt; use Piwik\Tracker\Visit\ReferrerSpamFilter; use Psr\Log\LoggerInterface; use Piwik\SettingsPiwik; @@ -67,6 +68,8 @@ class Tasks extends \Piwik\Plugin\Tasks // sure all archives that need to be invalidated get invalidated $this->daily('invalidateOutdatedArchives', null, self::HIGH_PRIORITY); + $this->daily('deleteOldFingerprintSalts', null, self::HIGH_PRIORITY); + // general data purge on older archive tables, executed daily $this->daily('purgeOutdatedArchives', null, self::HIGH_PRIORITY); @@ -88,6 +91,11 @@ class Tasks extends \Piwik\Plugin\Tasks $this->scheduleTrackingCodeReminderChecks(); } + public function deleteOldFingerprintSalts() + { + StaticContainer::get(FingerprintSalt::class)->deleteOldSalts(); + } + public function invalidateOutdatedArchives() { if (!Rules::isBrowserTriggerEnabled()) { diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php index 89c62252d5..b4e0c03d5a 100644 --- a/plugins/CoreAdminHome/tests/Integration/TasksTest.php +++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php @@ -133,6 +133,7 @@ class TasksTest extends IntegrationTestCase $expected = [ 'invalidateOutdatedArchives.', + 'deleteOldFingerprintSalts.', 'purgeOutdatedArchives.', 'purgeInvalidatedArchives.', 'purgeOrphanedArchives.', diff --git a/plugins/SitesManager/SitesManager.php b/plugins/SitesManager/SitesManager.php index 1320c0f6f2..2b8e767b26 100644 --- a/plugins/SitesManager/SitesManager.php +++ b/plugins/SitesManager/SitesManager.php @@ -13,12 +13,14 @@ use Piwik\API\Request; use Piwik\Common; use Piwik\Config; use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Option; use Piwik\Piwik; use Piwik\Plugins\CoreHome\SystemSummary; use Piwik\Settings\Storage\Backend\MeasurableSettingsTable; use Piwik\Tracker\Cache; +use Piwik\Tracker\FingerprintSalt; use Piwik\Tracker\Model as TrackerModel; use Piwik\Session\SessionNamespace; @@ -197,6 +199,16 @@ class SitesManager extends \Piwik\Plugin $array['timezone'] = $this->getTimezoneFromWebsite($website); $array['ts_created'] = $website['ts_created']; $array['type'] = $website['type']; + + // we make sure to have the fingerprint salts for the last 3 days incl tmrw in the cache so we don't need to + // query the DB directly for these days + $datesToGenerateSalt = array(Date::now()->addDay(1), Date::now(), Date::now()->subDay(1), Date::now()->subDay(2)); + + $fingerprintSaltKey = new FingerprintSalt(); + foreach ($datesToGenerateSalt as $date) { + $dateString = $fingerprintSaltKey->getDateString($date, $array['timezone']); + $array[FingerprintSalt::OPTION_PREFIX . $dateString] = $fingerprintSaltKey->getSalt($dateString, $idSite); + } } public function setTrackerCacheGeneral(&$cache) diff --git a/tests/PHPUnit/Integration/Tracker/FingerprintSaltTest.php b/tests/PHPUnit/Integration/Tracker/FingerprintSaltTest.php new file mode 100644 index 0000000000..4680ca3d00 --- /dev/null +++ b/tests/PHPUnit/Integration/Tracker/FingerprintSaltTest.php @@ -0,0 +1,109 @@ +<?php +/** + * Matomo - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ + +namespace Piwik\Tests\Integration\Tracker; + +use Piwik\Config; +use Piwik\Date; +use Piwik\Plugins\SitesManager\API; +use Piwik\Tests\Framework\Mock\FakeAccess; +use Piwik\Tracker\Action; +use Piwik\Tracker\FingerprintSalt; +use Piwik\Tracker\PageUrl; +use Piwik\Tracker\Request; +use Piwik\Translate; +use Piwik\Plugin\Manager as PluginManager; +use Piwik\Tests\Framework\TestCase\IntegrationTestCase; + +/** + * @group Core + * @group ActionTest + */ +class FingerprintSaltTest extends IntegrationTestCase +{ + /** + * @var FingerprintSalt + */ + private $fingerprintSalt; + + public function setUp() + { + parent::setUp(); + + $this->fingerprintSalt = new FingerprintSalt(); + } + + public function test_generateSalt() + { + $salt = $this->fingerprintSalt->generateSalt(); + $this->assertEquals(32, strlen($salt)); + $this->assertTrue(ctype_alnum($salt)); + } + + public function test_generateSalt_isRandom() + { + $this->assertNotSame($this->fingerprintSalt->generateSalt(), $this->fingerprintSalt->generateSalt()); + } + + public function test_getDateString() + { + $date = Date::factory('2020-05-05 14:04:05'); + $this->assertSame('2020-05-06',$this->fingerprintSalt->getDateString($date, 'Pacific/Auckland')); + $this->assertSame('2020-05-05',$this->fingerprintSalt->getDateString($date, 'Europe/Berlin')); + } + + public function test_getSalt_remembersSaltPerSite() + { + $salt05_1 = $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 1); + $salt06_1 = $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 1); + $salt05_2 = $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 2); + $salt06_2 = $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 2); + + $this->assertNotSame($salt05_1, $salt06_1); + $this->assertNotSame($salt05_2, $salt06_2); + $this->assertNotSame($salt06_1, $salt06_2); + + $this->assertSame($salt05_1, $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 1)); + $this->assertSame($salt06_1, $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 1)); + $this->assertSame($salt05_2, $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 2)); + } + + public function test_deleteOldSalts_whenNothingToDelete() + { + $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 1); + $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 1); + + Date::$now = time() - FingerprintSalt::DELETE_FINGERPRINT_OLDER_THAN_SECONDS + 30;// they would expire in 30 seconds + $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 2); + $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 2); + + Date::$now = time(); + $this->assertSame(array(), $this->fingerprintSalt->deleteOldSalts()); + } + + public function test_deleteOldSalts_someToBeDeleted() + { + $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 1); + + Date::$now = time() - FingerprintSalt::DELETE_FINGERPRINT_OLDER_THAN_SECONDS - 30; // these entries should be expired + $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 1); + $this->fingerprintSalt->getSalt('2020-05-05', $idSite = 2); + $this->fingerprintSalt->getSalt('2020-05-06', $idSite = 2); + + Date::$now = time(); + $this->assertSame(array( + 'fingerprint_salt_1_2020-05-06', + 'fingerprint_salt_2_2020-05-05', + 'fingerprint_salt_2_2020-05-06' + ), $this->fingerprintSalt->deleteOldSalts()); + + // executing it again wont delete anything + $this->assertSame(array(), $this->fingerprintSalt->deleteOldSalts()); + } + +} diff --git a/tests/PHPUnit/Unit/Config/IniFileChainTest.php b/tests/PHPUnit/Unit/Config/IniFileChainTest.php index 2d6c56a6d6..f8e88ea058 100644 --- a/tests/PHPUnit/Unit/Config/IniFileChainTest.php +++ b/tests/PHPUnit/Unit/Config/IniFileChainTest.php @@ -8,6 +8,7 @@ namespace Piwik\Tests\Unit\Config; use PHPUnit_Framework_TestCase; +use Piwik\Config; use Piwik\Config\IniFileChain; /** @@ -379,4 +380,36 @@ class IniFileChainTest extends PHPUnit_Framework_TestCase $actualOutput = $fileChain->dumpChanges($header); $this->assertEquals($expectedDumpChanges, $actualOutput); } + + + public function test_dump_handlesSpecialCharsCorrectly() + { + $config = new IniFileChain(); + $config->set('first', ["a[]\n\n[d]\n\nb=4" => "\n\n[def]\na=b"]); + $config->set('second', ["a[]\n\n[d]b=4" => 'b']); + $config->set('thir][d]', ['a' => 'b']); + $config->set("four]\n\n[def]\n", ['d[]' => 'e']); + $out = $config->dump(); + + $expected = <<<END +[first] +a[][d]b4 = " + +[def] +a=b" + +[second] +a[][d]b4 = "b" + +[third] +a = "b" + +[fourdef] +d[] = "e" + + +END; + + $this->assertEquals($expected, $out); + } }
\ No newline at end of file |