diff options
82 files changed, 1483 insertions, 939 deletions
diff --git a/core/Access.php b/core/Access.php index 874d95f254..d99551c219 100644 --- a/core/Access.php +++ b/core/Access.php @@ -727,6 +727,16 @@ class Access throw new NoAccessException($message); } + + /** + * Returns true if the current user is logged in or not. + * + * @return bool + */ + public function isUserLoggedIn() + { + return !empty($this->login); + } } /** diff --git a/core/Archive.php b/core/Archive.php index ad85d6242e..863b9e62e2 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -12,7 +12,6 @@ use Piwik\Archive\ArchiveQuery; use Piwik\Archive\ArchiveQueryFactory; use Piwik\Archive\Parameters; use Piwik\ArchiveProcessor\Rules; -use Piwik\Archive\ArchiveInvalidator; use Piwik\Container\StaticContainer; use Piwik\DataAccess\ArchiveSelector; @@ -168,11 +167,6 @@ class Archive implements ArchiveQuery private static $cache; /** - * @var ArchiveInvalidator - */ - private $invalidator; - - /** * @param Parameters $params * @param bool $forceIndexedBySite Whether to force index the result of a query by site ID. * @param bool $forceIndexedByDate Whether to force index the result of a query by period. @@ -183,8 +177,6 @@ class Archive implements ArchiveQuery $this->params = $params; $this->forceIndexedBySite = $forceIndexedBySite; $this->forceIndexedByDate = $forceIndexedByDate; - - $this->invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator'); } /** @@ -453,67 +445,6 @@ class Archive implements ArchiveQuery return $dataTable; } - private function getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet() - { - if (is_null(self::$cache)) { - self::$cache = Cache::getTransientCache(); - } - - $id = 'Archive.SiteIdsOfRememberedReportsInvalidated'; - - if (!self::$cache->contains($id)) { - self::$cache->save($id, array()); - } - - $siteIdsAlreadyHandled = self::$cache->fetch($id); - $siteIdsRequested = $this->params->getIdSites(); - - foreach ($siteIdsRequested as $index => $siteIdRequested) { - $siteIdRequested = (int) $siteIdRequested; - - if (in_array($siteIdRequested, $siteIdsAlreadyHandled)) { - unset($siteIdsRequested[$index]); // was already handled previously, do not do it again - } else { - $siteIdsAlreadyHandled[] = $siteIdRequested; // we will handle this id this time - } - } - - self::$cache->save($id, $siteIdsAlreadyHandled); - - return $siteIdsRequested; - } - - private function invalidatedReportsIfNeeded() - { - $siteIdsRequested = $this->getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet(); - - if (empty($siteIdsRequested)) { - return; // all requested site ids were already handled - } - - $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); - - foreach ($sitesPerDays as $date => $siteIds) { - if (empty($siteIds)) { - continue; - } - - $siteIdsToActuallyInvalidate = array_intersect($siteIds, $siteIdsRequested); - - if (empty($siteIdsToActuallyInvalidate)) { - continue; // all site ids that should be handled are already handled - } - - try { - $this->invalidator->markArchivesAsInvalidated($siteIdsToActuallyInvalidate, array(Date::factory($date)), false); - } catch (\Exception $e) { - Site::clearCache(); - throw $e; - } - } - - Site::clearCache(); - } /** * Queries archive tables for data and returns the result. @@ -638,8 +569,6 @@ class Archive implements ArchiveQuery */ private function cacheArchiveIdsAfterLaunching($archiveGroups, $plugins) { - $this->invalidatedReportsIfNeeded(); - $today = Date::today(); foreach ($this->params->getPeriods() as $period) { @@ -856,8 +785,11 @@ class Archive implements ArchiveQuery */ private function prepareArchive(array $archiveGroups, Site $site, Period $period) { + // if cron archiving is running, we will invalidate in CronArchive, not here + $invalidateBeforeArchiving = !SettingsServer::isArchivePhpTriggered(); + $parameters = new ArchiveProcessor\Parameters($site, $period, $this->params->getSegment()); - $archiveLoader = new ArchiveProcessor\Loader($parameters); + $archiveLoader = new ArchiveProcessor\Loader($parameters, $invalidateBeforeArchiving); $periodString = $period->getRangeString(); diff --git a/core/Archive/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php index 129db58151..d63c06180f 100644 --- a/core/Archive/ArchiveInvalidator.php +++ b/core/Archive/ArchiveInvalidator.php @@ -77,11 +77,12 @@ class ArchiveInvalidator // we do not really have to get the value first. we could simply always try to call set() and it would update or // insert the record if needed but we do not want to lock the table (especially since there are still some // MyISAM installations) - $values = Option::getLike($this->rememberArchivedReportIdStart . '%'); + $values = Option::getLike('%' . $this->rememberArchivedReportIdStart . '%'); $all = []; foreach ($values as $name => $value) { - $suffix = substr($name, strlen($this->rememberArchivedReportIdStart)); + $suffix = substr($name, strpos($name, $this->rememberArchivedReportIdStart)); + $suffix = str_replace($this->rememberArchivedReportIdStart, '', $suffix); list($idSite, $dateStr) = explode('_', $suffix); $all[$idSite][$dateStr] = $value; @@ -101,7 +102,7 @@ class ArchiveInvalidator // we do not really have to get the value first. we could simply always try to call set() and it would update or // insert the record if needed but we do not want to lock the table (especially since there are still some // MyISAM installations) - $value = Option::getLike($key . '%'); + $value = Option::getLike('%' . $key . '%'); } // getLike() returns an empty array rather than 'false' @@ -116,6 +117,7 @@ class ArchiveInvalidator $mykey = $this->buildRememberArchivedReportIdProcessSafe($idSite, $date->toString()); Option::set($mykey, '1'); Cache::clearCacheGeneral(); + return $mykey; } } @@ -133,16 +135,21 @@ class ArchiveInvalidator public function getRememberedArchivedReportsThatShouldBeInvalidated() { - $reports = Option::getLike($this->rememberArchivedReportIdStart . '%_%'); + $reports = Option::getLike('%' . $this->rememberArchivedReportIdStart . '%_%'); $sitesPerDay = array(); foreach ($reports as $report => $value) { + $report = substr($report, strpos($report, $this->rememberArchivedReportIdStart)); $report = str_replace($this->rememberArchivedReportIdStart, '', $report); $report = explode('_', $report); $siteId = (int) $report[0]; $date = $report[1]; + if (empty($siteId)) { + continue; + } + if (empty($sitesPerDay[$date])) { $sitesPerDay[$date] = array(); } @@ -169,14 +176,16 @@ class ArchiveInvalidator // This version is multi process safe on the insert of a new date to invalidate. private function buildRememberArchivedReportIdProcessSafe($idSite, $date) { - $id = $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date); + $id = Common::getRandomString(4, 'abcdefghijklmnoprstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ') . '_'; + $id .= $this->buildRememberArchivedReportIdForSiteAndDate($idSite, $date); $id .= '_' . Common::getProcessId(); + return $id; } public function forgetRememberedArchivedReportsToInvalidateForSite($idSite) { - $id = $this->buildRememberArchivedReportIdForSite($idSite) . '_%'; + $id = $this->buildRememberArchivedReportIdForSite($idSite); $this->deleteOptionLike($id); Cache::clearCacheGeneral(); } @@ -200,7 +209,7 @@ class ArchiveInvalidator { // we're not using deleteLike since it maybe could cause deadlocks see https://github.com/matomo-org/matomo/issues/15545 // we want to reduce number of rows scanned and only delete specific primary key - $keys = Option::getLike($id . '%'); + $keys = Option::getLike('%' . $id . '%'); if (empty($keys)) { return; diff --git a/core/ArchiveProcessor/Loader.php b/core/ArchiveProcessor/Loader.php index 4564146bd9..20627e98fd 100644 --- a/core/ArchiveProcessor/Loader.php +++ b/core/ArchiveProcessor/Loader.php @@ -8,13 +8,18 @@ */ namespace Piwik\ArchiveProcessor; +use Piwik\Archive\ArchiveInvalidator; use Piwik\Cache; use Piwik\Config; use Piwik\Container\StaticContainer; use Piwik\Context; use Piwik\DataAccess\ArchiveSelector; +use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; +use Piwik\Db; use Piwik\Piwik; +use Piwik\Site; +use Psr\Log\LoggerInterface; /** * This class uses PluginsArchiver class to trigger data aggregation and create archives. @@ -33,9 +38,28 @@ class Loader */ protected $params; - public function __construct(Parameters $params) + /** + * @var ArchiveInvalidator + */ + private $invalidator; + + /** + * @var \Piwik\Cache\Cache + */ + private $cache; + + /** + * @var LoggerInterface + */ + private $logger; + + public function __construct(Parameters $params, $invalidateBeforeArchiving = false) { $this->params = $params; + $this->invalidateBeforeArchiving = $invalidateBeforeArchiving; + $this->invalidator = StaticContainer::get(ArchiveInvalidator::class); + $this->cache = Cache::getTransientCache(); + $this->logger = StaticContainer::get(LoggerInterface::class); } /** @@ -65,11 +89,19 @@ class Loader { $this->params->setRequestedPlugin($pluginName); - list($idArchive, $visits, $visitsConverted) = $this->loadExistingArchiveIdFromDb(); - if (!empty($idArchive)) { + list($idArchive, $visits, $visitsConverted, $isAnyArchiveExists) = $this->loadExistingArchiveIdFromDb(); + if (!empty($idArchive)) { // we have a usable idarchive (it's not invalidated and it's new enough) return $idArchive; } + // if there is an archive, but we can't use it for some reason, invalidate existing archives before + // we start archiving. if the archive is made invalid, we will correctly re-archive below. + if ($this->invalidateBeforeArchiving + && $isAnyArchiveExists + ) { + $this->invalidatedReportsIfNeeded(); + } + /** @var ArchivingStatus $archivingStatus */ $archivingStatus = StaticContainer::get(ArchivingStatus::class); $archivingStatus->archiveStarted($this->params); @@ -170,20 +202,17 @@ class Loader */ public function loadExistingArchiveIdFromDb() { - $noArchiveFound = array(false, false, false); - if ($this->isArchivingForcedToTrigger()) { - return $noArchiveFound; - } + $this->logger->debug("Archiving forced to trigger for {$this->params}."); - $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed(); - $idAndVisits = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC); - - if (!$idAndVisits) { - return $noArchiveFound; + // return no usable archive found, and no existing archive. this will skip invalidation, which should + // be fine since we just force archiving. + return [false, false, false, false]; } - return $idAndVisits; + $minDatetimeArchiveProcessedUTC = $this->getMinTimeArchiveProcessed(); + $result = ArchiveSelector::getArchiveIdAndVisits($this->params, $minDatetimeArchiveProcessedUTC); + return $result; } /** @@ -242,4 +271,46 @@ class Loader return $cache->fetch($cacheKey); } + + // public for tests + public function getReportsToInvalidate() + { + $sitesPerDays = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + foreach ($sitesPerDays as $dateStr => $siteIds) { + if (empty($siteIds) + || !in_array($this->params->getSite()->getId(), $siteIds) + ) { + unset($sitesPerDays[$dateStr]); + } + + $date = Date::factory($dateStr); + if ($date->isEarlier($this->params->getPeriod()->getDateStart()) + || $date->isLater($this->params->getPeriod()->getDateEnd()) + ) { // date in list is not the current date, so ignore it + unset($sitesPerDays[$dateStr]); + } + } + + return $sitesPerDays; + } + + private function invalidatedReportsIfNeeded() + { + $sitesPerDays = $this->getReportsToInvalidate(); + if (empty($sitesPerDays)) { + return; + } + + foreach ($sitesPerDays as $date => $siteIds) { + try { + $this->invalidator->markArchivesAsInvalidated([$this->params->getSite()->getId()], array(Date::factory($date)), false, $this->params->getSegment()); + } catch (\Exception $e) { + Site::clearCache(); + throw $e; + } + } + + Site::clearCache(); + } } diff --git a/core/ArchiveProcessor/Parameters.php b/core/ArchiveProcessor/Parameters.php index e765787920..954be0a87d 100644 --- a/core/ArchiveProcessor/Parameters.php +++ b/core/ArchiveProcessor/Parameters.php @@ -258,4 +258,9 @@ class Parameters { $this->isRootArchiveRequest = $isRootArchiveRequest; } + + public function __toString() + { + return "[idSite = {$this->getSite()->getId()}, period = {$this->getPeriod()->getLabel()} {$this->getPeriod()->getRangeString()}, segment = {$this->getSegment()->getString()}]"; + } } diff --git a/core/Config/IniFileChain.php b/core/Config/IniFileChain.php index 7e064b0dad..ff9798762d 100644 --- a/core/Config/IniFileChain.php +++ b/core/Config/IniFileChain.php @@ -210,13 +210,16 @@ class IniFileChain $this->resetSettingsChain($defaultSettingsFiles, $userSettingsFile); } - if (!empty($userSettingsFile) && !empty($GLOBALS['ENABLE_CONFIG_PHP_CACHE'])) { + $hasAbsoluteConfigFile = !empty($userSettingsFile) && strpos($userSettingsFile, DIRECTORY_SEPARATOR) === 0; + $useConfigCache = !empty($GLOBALS['ENABLE_CONFIG_PHP_CACHE']) && $hasAbsoluteConfigFile; + + if ($useConfigCache) { $cache = new Cache(); $values = $cache->doFetch(self::CONFIG_CACHE_KEY); if (!empty($values) && isset($values['mergedSettings']) - && isset($values['settingsChain'])) { + && isset($values['settingsChain'][$userSettingsFile])) { $this->mergedSettings = $values['mergedSettings']; $this->settingsChain = $values['settingsChain']; return; @@ -246,8 +249,7 @@ class IniFileChain $this->mergedSettings = call_user_func($GLOBALS['MATOMO_MODIFY_CONFIG_SETTINGS'], $this->mergedSettings); } - if (!empty($GLOBALS['ENABLE_CONFIG_PHP_CACHE']) - && !empty($userSettingsFile) + if ($useConfigCache && !empty($this->mergedSettings) && !empty($this->settingsChain)) { diff --git a/core/Cookie.php b/core/Cookie.php index 26cbfe6a3f..6cecd6de53 100644 --- a/core/Cookie.php +++ b/core/Cookie.php @@ -441,16 +441,18 @@ class Cookie $sameSite = ucfirst(strtolower($default)); if ($sameSite == 'None') { - $userAgent = Http::getUserAgent(); - $ddFactory = StaticContainer::get(\Piwik\DeviceDetector\DeviceDetectorFactory::class); - $deviceDetector = $ddFactory->makeInstance($userAgent); - $deviceDetector->parse(); - - $browserFamily = \DeviceDetector\Parser\Client\Browser::getBrowserFamily($deviceDetector->getClient('short_name')); - if ((!ProxyHttp::isHttps()) && $browserFamily === 'Chrome') { - $sameSite = 'Lax'; - } else if ($browserFamily === 'Safari') { - $sameSite = ''; + if ((!ProxyHttp::isHttps())) { + $sameSite = 'Lax'; // None can be only used when secure flag will be set + } else { + $userAgent = Http::getUserAgent(); + $ddFactory = StaticContainer::get(\Piwik\DeviceDetector\DeviceDetectorFactory::class); + $deviceDetector = $ddFactory->makeInstance($userAgent); + $deviceDetector->parse(); + + $browserFamily = \DeviceDetector\Parser\Client\Browser::getBrowserFamily($deviceDetector->getClient('short_name')); + if ($browserFamily === 'Safari') { + $sameSite = ''; + } } } diff --git a/core/CronArchive.php b/core/CronArchive.php index 092aa4217c..bb1337dbb0 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -897,6 +897,8 @@ class CronArchive $visitsLastDays = 0; + $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, 'day', $date, $segment = ''); if ($isThereArchive) { $visitsToday = Archive::build($idSite, 'day', $date)->getNumeric('nb_visits'); @@ -971,7 +973,8 @@ class CronArchive return $dayArchiveWasSuccessful; } - private function isThereAValidArchiveForPeriod($idSite, $period, $date, $segment = '') + // public for tests + public function isThereAValidArchiveForPeriod($idSite, $period, $date, $segment = '') { $this->disconnectDb(); @@ -982,9 +985,17 @@ class CronArchive $periodsToCheck = [Factory::build($period, $date, Site::getTimezoneFor($idSite))]; } - $periodsToCheckRanges = array_map(function (Period $p) { return $p->getRangeString(); }, $periodsToCheck); + $isTodayIncluded = $this->isTodayIncludedInPeriod($idSite, $periodsToCheck); + $isLast = preg_match('/^last([0-9]+)/', $date, $matches); - $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + // don't do this check for a single period that includes today + if ($isTodayIncluded + && !$isLast + ) { + return [false, null]; + } + + $periodsToCheckRanges = array_map(function (Period $p) { return $p->getRangeString(); }, $periodsToCheck); $archiveIds = ArchiveSelector::getArchiveIds( [$idSite], $periodsToCheck, new Segment($segment, [$idSite]), $plugins = [], // empty plugins param since we only check for an 'all' archive @@ -1004,37 +1015,60 @@ class CronArchive // if there is an invalidated archive within the range, find out the oldest one and how far it is from today, // and change the lastN $date to be value so it is correctly re-processed. $newDate = $date; - if (!$isThereArchiveForAllPeriods - && preg_match('/^last([0-9]+)/', $date, $matches) - ) { - $lastNValue = (int) $matches[1]; - - usort($diff, function ($lhs, $rhs) { - $lhsDate = explode(',', $lhs)[0]; - $rhsDate = explode(',', $rhs)[0]; - - if ($lhsDate == $rhsDate) { - return 1; - } else if (Date::factory($lhsDate)->isEarlier(Date::factory($rhsDate))) { - return -1; - } else { - return 1; - } - }); + if ($isLast) { + if (!$isThereArchiveForAllPeriods) { + $lastNValue = (int)$matches[1]; + + usort($diff, function ($lhs, $rhs) { + $lhsDate = explode(',', $lhs)[0]; + $rhsDate = explode(',', $rhs)[0]; + + if ($lhsDate == $rhsDate) { + return 1; + } else if (Date::factory($lhsDate)->isEarlier(Date::factory($rhsDate))) { + return -1; + } else { + return 1; + } + }); - $oldestDateWithoutArchive = explode(',', reset($diff))[0]; - $todayInTimezone = Date::factoryInTimezone('today', Site::getTimezoneFor($idSite)); + $oldestDateWithoutArchive = explode(',', reset($diff))[0]; + $todayInTimezone = Date::factoryInTimezone('today', Site::getTimezoneFor($idSite)); - /** @var Range $newRangePeriod */ - $newRangePeriod = PeriodFactory::build($period, $oldestDateWithoutArchive . ',' . $todayInTimezone); + /** @var Range $newRangePeriod */ + $newRangePeriod = PeriodFactory::build($period, $oldestDateWithoutArchive . ',' . $todayInTimezone); - $newDate = 'last' . min($lastNValue, $newRangePeriod->getNumberOfSubperiods()); + $newDate = 'last' . max(min($lastNValue, $newRangePeriod->getNumberOfSubperiods()), 2); + } else if ($isTodayIncluded) { + $isThereArchiveForAllPeriods = false; + $newDate = 'last2'; + } } return [$isThereArchiveForAllPeriods, $newDate]; } /** + * @param int $idSite + * @param Period[] $periods + * @return bool + * @throws Exception + */ + private function isTodayIncludedInPeriod($idSite, $periods) + { + $timezone = Site::getTimezoneFor($idSite); + $today = Date::factoryInTimezone('today', $timezone); + + foreach ($periods as $period) { + if ($period->isDateInPeriod($today)) { + return true; + } + } + + return false; + } + + /** * @param $idSite * @return array */ @@ -1103,6 +1137,8 @@ class CronArchive return Request::ABORT; } + $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, $period, $date, $segment); if ($isThereArchive) { $this->logArchiveWebsiteSkippedValidArchiveExists($idSite, $period, $date); @@ -1993,6 +2029,8 @@ class CronArchive return Request::ABORT; } + $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + list($isThereArchive, $newDate) = $this->isThereAValidArchiveForPeriod($idSite, $period, $date, $segment); if ($isThereArchive) { $this->logArchiveWebsiteSkippedValidArchiveExists($idSite, $period, $date, $segment); diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php index 4232404141..daaeb7b2a6 100644 --- a/core/DataAccess/ArchiveSelector.php +++ b/core/DataAccess/ArchiveSelector.php @@ -48,7 +48,12 @@ class ArchiveSelector /** * @param ArchiveProcessor\Parameters $params * @param bool $minDatetimeArchiveProcessedUTC deprecated. will be removed in Matomo 4. - * @return array|bool + * @return array An array with four values: \ + * - the latest archive ID or false if none + * - the latest visits value for the latest archive, regardless of whether the archive is invalidated or not + * - the latest visits converted value for the latest archive, regardless of whether the archive is invalidated or not + * - whether there is an archive that exists or not. if this is true and the latest archive is false, it means + * the archive found was not usable (for example, it was invalidated and we are not looking for invalidated archives) * @throws Exception */ public static function getArchiveIdAndVisits(ArchiveProcessor\Parameters $params, $minDatetimeArchiveProcessedUTC = false, $includeInvalidated = true) @@ -61,80 +66,41 @@ class ArchiveSelector $numericTable = ArchiveTableCreator::getNumericTable($dateStart); - $minDatetimeIsoArchiveProcessedUTC = null; - if ($minDatetimeArchiveProcessedUTC) { - $minDatetimeIsoArchiveProcessedUTC = Date::factory($minDatetimeArchiveProcessedUTC)->getDatetime(); - } - $requestedPlugin = $params->getRequestedPlugin(); $segment = $params->getSegment(); $plugins = array("VisitsSummary", $requestedPlugin); $doneFlags = Rules::getDoneFlags($plugins, $segment); + $requestedPluginDoneFlags = Rules::getDoneFlags([$requestedPlugin], $segment); $doneFlagValues = Rules::getSelectableDoneFlagValues($includeInvalidated, $params); - $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues); - - if (empty($results)) { - return false; - } - - $idArchive = self::getMostRecentIdArchiveFromResults($segment, $requestedPlugin, $results); - - $idArchiveVisitsSummary = self::getMostRecentIdArchiveFromResults($segment, "VisitsSummary", $results); - - list($visits, $visitsConverted) = self::getVisitsMetricsFromResults($idArchive, $idArchiveVisitsSummary, $results); - - if (false === $visits && false === $idArchive) { - return false; + $results = self::getModel()->getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, null, $doneFlags); + if (empty($results)) { // no archive found + return [false, false, false, false]; } - return array($idArchive, $visits, $visitsConverted); - } + $result = self::findArchiveDataWithLatestTsArchived($results, $requestedPluginDoneFlags); - protected static function getVisitsMetricsFromResults($idArchive, $idArchiveVisitsSummary, $results) - { - $visits = $visitsConverted = false; - $archiveWithVisitsMetricsWasFound = ($idArchiveVisitsSummary !== false); + $visits = isset($result['nb_visits']) ? $result['nb_visits'] : false; + $visitsConverted = isset($result['nb_visits_converted']) ? $result['nb_visits_converted'] : false; - if ($archiveWithVisitsMetricsWasFound) { - $visits = $visitsConverted = 0; + if (isset($result['value']) + && !in_array($result['value'], $doneFlagValues) + ) { // the archive cannot be considered valid for this request (has wrong done flag value) + return [false, $visits, $visitsConverted, true]; } - foreach ($results as $result) { - if (in_array($result['idarchive'], array($idArchive, $idArchiveVisitsSummary))) { - $value = (int)$result['value']; - if (empty($visits) - && $result['name'] == self::NB_VISITS_RECORD_LOOKED_UP - ) { - $visits = $value; - } - if (empty($visitsConverted) - && $result['name'] == self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP - ) { - $visitsConverted = $value; - } - } + // the archive is too old + if ($minDatetimeArchiveProcessedUTC + && isset($result['idarchive']) + && Date::factory($result['ts_archived'])->isEarlier(Date::factory($minDatetimeArchiveProcessedUTC)) + ) { + return [false, $visits, $visitsConverted, true]; } - return array($visits, $visitsConverted); - } - - protected static function getMostRecentIdArchiveFromResults(Segment $segment, $requestedPlugin, $results) - { - $idArchive = false; - $namesRequestedPlugin = Rules::getDoneFlags(array($requestedPlugin), $segment); - - foreach ($results as $result) { - if ($idArchive === false - && in_array($result['name'], $namesRequestedPlugin) - ) { - $idArchive = $result['idarchive']; - break; - } - } + $idArchive = isset($result['idarchive']) ? $result['idarchive'] : false; - return $idArchive; + return array($idArchive, $visits, $visitsConverted, true); } /** @@ -213,7 +179,6 @@ class ArchiveSelector $sql = sprintf($getArchiveIdsSql, $table, $dateCondition); - $archiveIds = $db->fetchAll($sql, $bind); // get the archive IDs @@ -381,4 +346,71 @@ class ArchiveSelector // create the SQL to find archives that are DONE return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))"; } + + /** + * This method takes the output of Model::getArchiveIdAndVisits() and selects data from the + * latest archives. + * + * This includes: + * - the idarchive with the latest ts_archived ($results will be ordered by ts_archived desc) + * - the visits/converted visits of the latest archive, which includes archives for VisitsSummary alone + * ($requestedPluginDoneFlags will have the done flag for the overall archive plus a done flag for + * VisitsSummary by itself) + * - the ts_archived for the latest idarchive + * - the doneFlag value for the latest archive + * + * @param $results + * @param $requestedPluginDoneFlags + * @return array + */ + private static function findArchiveDataWithLatestTsArchived($results, $requestedPluginDoneFlags) + { + // find latest idarchive for each done flag + $idArchives = []; + foreach ($results as $row) { + $doneFlag = $row['name']; + if (preg_match('/^done/', $doneFlag) + && !isset($idArchives[$doneFlag]) + ) { + $idArchives[$doneFlag] = $row['idarchive']; + } + } + + $archiveData = []; + + // gather the latest visits/visits_converted metrics + foreach ($results as $row) { + $name = $row['name']; + if (!isset($archiveData[$name]) + && in_array($name, [self::NB_VISITS_RECORD_LOOKED_UP, self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP]) + && in_array($row['idarchive'], $idArchives) + ) { + $archiveData[$name] = $row['value']; + } + } + + // if an archive is found, but the metric data isn't found, we set the value to 0, + // so it won't get returned as false. this is here because the code used to do this before this change + // and we didn't want to introduce any side effects. it may be removable in the future. + foreach ([self::NB_VISITS_RECORD_LOOKED_UP, self::NB_VISITS_CONVERTED_RECORD_LOOKED_UP] as $metric) { + if (!empty($idArchives) + && !isset($archiveData[$metric]) + ) { + $archiveData[$metric] = 0; + } + } + + // set the idarchive & ts_archived for the archive we're looking for + foreach ($results as $row) { + $name = $row['name']; + if (in_array($name, $requestedPluginDoneFlags)) { + $archiveData['idarchive'] = $row['idarchive']; + $archiveData['ts_archived'] = $row['ts_archived']; + $archiveData['value'] = $row['value']; + break; + } + } + + return $archiveData; + } } diff --git a/core/DataAccess/LogAggregator.php b/core/DataAccess/LogAggregator.php index 0bfc099476..08111237a1 100644 --- a/core/DataAccess/LogAggregator.php +++ b/core/DataAccess/LogAggregator.php @@ -16,6 +16,7 @@ use Piwik\Container\StaticContainer; use Piwik\DataArray; use Piwik\Date; use Piwik\Db; +use Piwik\DbHelper; use Piwik\Metrics; use Piwik\Plugin\LogTablesProvider; use Piwik\Segment; @@ -462,18 +463,18 @@ class LogAggregator * * The following columns are in each row of the result set: * - * - **{@link Piwik\Metrics::INDEX_NB_UNIQ_VISITORS}**: The total number of unique visitors in this group + * - **{@link \Piwik\Metrics::INDEX_NB_UNIQ_VISITORS}**: The total number of unique visitors in this group * of aggregated visits. - * - **{@link Piwik\Metrics::INDEX_NB_VISITS}**: The total number of visits aggregated. - * - **{@link Piwik\Metrics::INDEX_NB_ACTIONS}**: The total number of actions performed in this group of + * - **{@link \Piwik\Metrics::INDEX_NB_VISITS}**: The total number of visits aggregated. + * - **{@link \Piwik\Metrics::INDEX_NB_ACTIONS}**: The total number of actions performed in this group of * aggregated visits. - * - **{@link Piwik\Metrics::INDEX_MAX_ACTIONS}**: The maximum actions perfomred in one visit for this group of + * - **{@link \Piwik\Metrics::INDEX_MAX_ACTIONS}**: The maximum actions perfomred in one visit for this group of * visits. - * - **{@link Piwik\Metrics::INDEX_SUM_VISIT_LENGTH}**: The total amount of time spent on the site for this + * - **{@link \Piwik\Metrics::INDEX_SUM_VISIT_LENGTH}**: The total amount of time spent on the site for this * group of visits. - * - **{@link Piwik\Metrics::INDEX_BOUNCE_COUNT}**: The total number of bounced visits in this group of + * - **{@link \Piwik\Metrics::INDEX_BOUNCE_COUNT}**: The total number of bounced visits in this group of * visits. - * - **{@link Piwik\Metrics::INDEX_NB_VISITS_CONVERTED}**: The total number of visits for which at least one + * - **{@link \Piwik\Metrics::INDEX_NB_VISITS_CONVERTED}**: The total number of visits for which at least one * conversion occurred, for this group of visits. * * Additional data can be selected by setting the `$additionalSelects` parameter. @@ -493,24 +494,26 @@ class LogAggregator * @param bool|array $metrics The set of metrics to calculate and return. If false, the query will select * all of them. The following values can be used: * - * - {@link Piwik\Metrics::INDEX_NB_UNIQ_VISITORS} - * - {@link Piwik\Metrics::INDEX_NB_VISITS} - * - {@link Piwik\Metrics::INDEX_NB_ACTIONS} - * - {@link Piwik\Metrics::INDEX_MAX_ACTIONS} - * - {@link Piwik\Metrics::INDEX_SUM_VISIT_LENGTH} - * - {@link Piwik\Metrics::INDEX_BOUNCE_COUNT} - * - {@link Piwik\Metrics::INDEX_NB_VISITS_CONVERTED} + * - {@link \Piwik\Metrics::INDEX_NB_UNIQ_VISITORS} + * - {@link \Piwik\Metrics::INDEX_NB_VISITS} + * - {@link \Piwik\Metrics::INDEX_NB_ACTIONS} + * - {@link \Piwik\Metrics::INDEX_MAX_ACTIONS} + * - {@link \Piwik\Metrics::INDEX_SUM_VISIT_LENGTH} + * - {@link \Piwik\Metrics::INDEX_BOUNCE_COUNT} + * - {@link \Piwik\Metrics::INDEX_NB_VISITS_CONVERTED} * @param bool|\Piwik\RankingQuery $rankingQuery * A pre-configured ranking query instance that will be used to limit the result. - * If set, the return value is the array returned by {@link Piwik\RankingQuery::execute()}. + * If set, the return value is the array returned by {@link \Piwik\RankingQuery::execute()}. + * @param bool|string $orderBy Order By clause to add (e.g. user_id ASC) + * @param int $timeLimitInMs Adds a MAX_EXECUTION_TIME query hint to the query if $timeLimitInMs > 0 * * @return mixed A Zend_Db_Statement if `$rankingQuery` isn't supplied, otherwise the result of - * {@link Piwik\RankingQuery::execute()}. Read {@link queryVisitsByDimension() this} + * {@link \Piwik\RankingQuery::execute()}. Read {@link queryVisitsByDimension() this} * to see what aggregate data is calculated by the query. * @api */ public function queryVisitsByDimension(array $dimensions = array(), $where = false, array $additionalSelects = array(), - $metrics = false, $rankingQuery = false, $orderBy = false) + $metrics = false, $rankingQuery = false, $orderBy = false, $timeLimitInMs = -1) { $tableName = self::LOG_VISIT_TABLE; $availableMetrics = $this->getVisitsMetricFields(); @@ -546,9 +549,11 @@ class LogAggregator $rankingQuery->addColumn(Metrics::INDEX_MAX_ACTIONS, 'max'); } - return $rankingQuery->execute($query['sql'], $query['bind']); + return $rankingQuery->execute($query['sql'], $query['bind'], $timeLimitInMs); } + $query['sql'] = DbHelper::addMaxExecutionTimeHintToQuery($query['sql'], $timeLimitInMs); + return $this->getDb()->query($query['sql'], $query['bind']); } @@ -873,6 +878,7 @@ class LogAggregator * If a string is used for this parameter, the table alias is not * suffixed (since there is only one column). * @param string $secondaryOrderBy A secondary order by clause for the ranking query + * @param int $timeLimitInMs Adds a MAX_EXECUTION_TIME hint to the query if $timeLimitInMs > 0 * @return mixed A Zend_Db_Statement if `$rankingQuery` isn't supplied, otherwise the result of * {@link Piwik\RankingQuery::execute()}. Read [this](#queryEcommerceItems-result-set) * to see what aggregate data is calculated by the query. @@ -885,7 +891,8 @@ class LogAggregator $metrics = false, $rankingQuery = null, $joinLogActionOnColumn = false, - $secondaryOrderBy = null + $secondaryOrderBy = null, + $timeLimitInMs = -1 ) { $tableName = self::LOG_ACTIONS_TABLE; $availableMetrics = $this->getActionsMetricFields(); @@ -937,9 +944,11 @@ class LogAggregator $rankingQuery->addColumn($sumColumns, 'sum'); - return $rankingQuery->execute($query['sql'], $query['bind']); + return $rankingQuery->execute($query['sql'], $query['bind'], $timeLimitInMs); } + $query['sql'] = DbHelper::addMaxExecutionTimeHintToQuery($query['sql'], $timeLimitInMs); + return $this->getDb()->query($query['sql'], $query['bind']); } diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index ab32487f8d..4e72698269 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -215,7 +215,8 @@ class Model return $deletedRows; } - public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues) + public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, + $doneFlags, $doneFlagValues = null) { $bindSQL = array($idSite, $dateStartIso, @@ -231,7 +232,8 @@ class Model $bindSQL[] = $minDatetimeIsoArchiveProcessedUTC; } - $sqlQuery = "SELECT idarchive, value, name, date1 as startDate FROM $numericTable + // NOTE: we can't predict how many segments there will be so there could be lots of nb_visits/nb_visits_converted rows... have to select everything. + $sqlQuery = "SELECT idarchive, value, name, ts_archived, date1 as startDate FROM $numericTable WHERE idsite = ? AND date1 = ? AND date2 = ? @@ -240,7 +242,7 @@ class Model OR name = '" . ArchiveSelector::NB_VISITS_RECORD_LOOKED_UP . "' OR name = '" . ArchiveSelector::NB_VISITS_CONVERTED_RECORD_LOOKED_UP . "') $timeStampWhere - ORDER BY idarchive DESC"; + ORDER BY ts_archived DESC, idarchive DESC"; $results = Db::fetchAll($sqlQuery, $bindSQL); return $results; @@ -428,7 +430,14 @@ class Model $allDoneFlags = "'" . implode("','", $doneFlags) . "'"; // create the SQL to find archives that are DONE - return "((name IN ($allDoneFlags)) AND (value IN (" . implode(',', $possibleValues) . ")))"; + $result = "((name IN ($allDoneFlags))"; + + if (!empty($possibleValues)) { + $result .= " AND (value IN (" . implode(',', $possibleValues) . ")))"; + } + $result .= ')'; + + return $result; } } diff --git a/core/DbHelper.php b/core/DbHelper.php index b8cf2509ea..6939dae660 100644 --- a/core/DbHelper.php +++ b/core/DbHelper.php @@ -178,6 +178,21 @@ class DbHelper } /** + * Returns if the given table has an index with the given name + * + * @param string $table + * @param string $indexName + * + * @return bool + * @throws Exception + */ + public static function tableHasIndex($table, $indexName) + { + $result = Db::get()->fetchOne('SHOW INDEX FROM '.$table.' WHERE Key_name = ?', [$indexName]); + return !empty($result); + } + + /** * Get the SQL to create Piwik tables * * @return array array of strings containing SQL @@ -213,6 +228,33 @@ class DbHelper } /** + * Adds a MAX_EXECUTION_TIME hint into a SELECT query if $limit is bigger than 1 + * + * @param string $sql query to add hint to + * @param int $limit time limit in seconds + * @return string + */ + public static function addMaxExecutionTimeHintToQuery($sql, $limit) + { + if ($limit <= 0) { + return $sql; + } + + $sql = trim($sql); + $pos = stripos($sql, 'SELECT'); + if ($pos !== false) { + + $timeInMs = $limit * 1000; + $timeInMs = (int) $timeInMs; + $maxExecutionTimeHint = ' /*+ MAX_EXECUTION_TIME('.$timeInMs.') */ '; + + $sql = substr_replace($sql, 'SELECT ' . $maxExecutionTimeHint, $pos, strlen('SELECT')); + } + + return $sql; + } + + /** * Returns true if the string is a valid database name for MySQL. MySQL allows + in the database names. * Database names that start with a-Z or 0-9 and contain a-Z, 0-9, underscore(_), dash(-), plus(+), and dot(.) will be accepted. * File names beginning with anything but a-Z or 0-9 will be rejected (including .htaccess for example). diff --git a/core/FrontController.php b/core/FrontController.php index 84ce6a1927..0ecd562fad 100644 --- a/core/FrontController.php +++ b/core/FrontController.php @@ -392,6 +392,7 @@ class FrontController extends Singleton $loggedIn = false; + // don't use sessionauth in cli mode // try authenticating w/ session first... $sessionAuth = $this->makeSessionAuthenticator(); if ($sessionAuth) { @@ -647,6 +648,12 @@ class FrontController extends Singleton private function makeSessionAuthenticator() { + if (Common::isPhpClimode() + && !defined('PIWIK_TEST_MODE') + ) { // don't use the session auth during CLI requests + return null; + } + $module = Common::getRequestVar('module', self::DEFAULT_MODULE, 'string'); $action = Common::getRequestVar('action', false); diff --git a/core/Http.php b/core/Http.php index f7dc89f9de..063379a727 100644 --- a/core/Http.php +++ b/core/Http.php @@ -934,9 +934,12 @@ class Http * With HTTP/2 Cloudflare is passing headers in lowercase (e.g. 'content-type' instead of 'Content-Type') * which breaks any code which uses the header data. */ - $camelName = ucwords($name, '-'); - if ($camelName !== $name) { - $headers[$camelName] = trim($value); + if (version_compare(PHP_VERSION, '5.5.16', '>=')) { + // Passing a second arg to ucwords is not supported by older versions of PHP + $camelName = ucwords($name, '-'); + if ($camelName !== $name) { + $headers[$camelName] = trim($value); + } } } diff --git a/core/Period.php b/core/Period.php index 6e3ef0f327..e717995f28 100644 --- a/core/Period.php +++ b/core/Period.php @@ -248,6 +248,21 @@ abstract class Period } /** + * Returns whether the date `$date` is within the current period or not. + * + * Note: the time component of the period's dates and `$date` is ignored. + * + * @param Date $today + * @return bool + */ + public function isDateInPeriod(Date $date) + { + $ts = $date->getStartOfDay()->getTimestamp(); + return $ts >= $this->getDateStart()->getStartOfDay()->getTimestamp() + && $ts < $this->getDateEnd()->addDay(1)->getStartOfDay()->getTimestamp(); + } + + /** * Add a date to the period. * * Protected because adding periods after initialization is not supported. diff --git a/core/Period/Factory.php b/core/Period/Factory.php index 1f4afabbc2..4fac7eba51 100644 --- a/core/Period/Factory.php +++ b/core/Period/Factory.php @@ -74,6 +74,7 @@ abstract class Factory self::checkPeriodIsEnabled($period); if (is_string($date)) { + list($period, $date) = self::convertRangeToDateIfNeeded($period, $date); if (Period::isMultiplePeriod($date, $period) || $period == 'range' ) { @@ -130,6 +131,19 @@ abstract class Factory throw new Exception($message); } + private static function convertRangeToDateIfNeeded($period, $date) + { + if (is_string($period) && is_string($date) && $period === 'range') { + $dates = explode(',', $date); + if (count($dates) === 2 && $dates[0] === $dates[1]) { + $period = 'day'; + $date = $dates[0]; + } + } + + return array($period, $date); + } + /** * Creates a Period instance using a period, date and timezone. * @@ -146,6 +160,8 @@ abstract class Factory $timezone = 'UTC'; } + list($period, $date) = self::convertRangeToDateIfNeeded($period, $date); + if ($period == 'range') { self::checkPeriodIsEnabled('range'); $oPeriod = new Range('range', $date, $timezone, Date::factory('today', $timezone)); diff --git a/core/Plugin/ControllerAdmin.php b/core/Plugin/ControllerAdmin.php index be2f5a977e..87c0873dd0 100644 --- a/core/Plugin/ControllerAdmin.php +++ b/core/Plugin/ControllerAdmin.php @@ -207,12 +207,12 @@ abstract class ControllerAdmin extends Controller } /** - * PHP Version required by the next major Piwik version + * PHP Version required by the next major Matomo version * @return string */ private static function getNextRequiredMinimumPHP() { - return '7.1'; + return '7.2'; } private static function isUsingPhpVersionCompatibleWithNextPiwik() @@ -222,8 +222,6 @@ abstract class ControllerAdmin extends Controller private static function notifyWhenPhpVersionIsNotCompatibleWithNextMajorPiwik() { - return; // no major version coming - if (self::isUsingPhpVersionCompatibleWithNextPiwik()) { return; } diff --git a/core/RankingQuery.php b/core/RankingQuery.php index c784000393..c0b4571015 100644 --- a/core/RankingQuery.php +++ b/core/RankingQuery.php @@ -220,12 +220,15 @@ class RankingQuery * has to be specified in this query. {@link RankingQuery} cannot apply ordering * itself. * @param $bind array Bindings for the inner query. + * @param int $timeLimitInMs Adds a MAX_EXECUTION_TIME query hint to the query if $timeLimitInMs > 0 * @return array The format depends on which methods have been used * to configure the ranking query. */ - public function execute($innerQuery, $bind = array()) + public function execute($innerQuery, $bind = array(), $timeLimitInMs = 0) { $query = $this->generateRankingQuery($innerQuery); + $query = DbHelper::addMaxExecutionTimeHintToQuery($query, $timeLimitInMs); + $data = Db::getReader()->fetchAll($query, $bind); if ($this->columnToMarkExcludedRows !== false) { diff --git a/core/Session.php b/core/Session.php index bed2df45bc..5b73ef961a 100644 --- a/core/Session.php +++ b/core/Session.php @@ -207,7 +207,7 @@ class Session extends Zend_Session { $headerStr = 'Set-Cookie: ' . rawurlencode($name) . '=' . rawurlencode($value); if ($expires) { - $headerStr .= '; expires=' . $expires; + $headerStr .= '; expires=' . gmdate('D, d-M-Y H:i:s', $expires) . ' GMT'; } if ($path) { $headerStr .= '; path=' . $path; diff --git a/core/Tracker/Model.php b/core/Tracker/Model.php index 74ef5b3430..2735d6e4a4 100644 --- a/core/Tracker/Model.php +++ b/core/Tracker/Model.php @@ -16,6 +16,7 @@ use Psr\Log\LoggerInterface; class Model { + const CACHE_KEY_INDEX_IDSITE_IDVISITOR = 'log_visit_has_index_idsite_idvisitor'; public function createAction($visitAction) { @@ -70,10 +71,10 @@ class Model $sqlBind[] = $value; } - $parts = implode($updateParts, ', '); + $parts = implode(', ', $updateParts); $table = Common::prefixTable('log_conversion'); - $sql = "UPDATE $table SET $parts WHERE " . implode($updateWhereParts, ' AND '); + $sql = "UPDATE $table SET $parts WHERE " . implode(' AND ', $updateWhereParts); try { $this->getDb()->query($sql, $sqlBind); @@ -285,7 +286,7 @@ class Model $sqlBind[] = $value; } - $parts = implode($updateParts, ', '); + $parts = implode(', ', $updateParts); $table = Common::prefixTable('log_conversion_item'); $sql = "UPDATE $table SET $parts WHERE idvisit = ? AND idorder = ? AND idaction_sku = ?"; @@ -428,7 +429,13 @@ class Model private function findVisitorByVisitorId($idVisitor, $select, $from, $where, $bindSql) { - // will use INDEX index_idsite_idvisitor (idsite, idvisitor) + $cache = Cache::getCacheGeneral(); + + // use INDEX index_idsite_idvisitor (idsite, idvisitor) if available + if (array_key_exists(self::CACHE_KEY_INDEX_IDSITE_IDVISITOR, $cache) && true === $cache[self::CACHE_KEY_INDEX_IDSITE_IDVISITOR]) { + $from .= ' FORCE INDEX (index_idsite_idvisitor) '; + } + $where .= ' AND idvisitor = ?'; $bindSql[] = $idVisitor; diff --git a/core/Tracker/Request.php b/core/Tracker/Request.php index f33bed656d..7da7f12287 100644 --- a/core/Tracker/Request.php +++ b/core/Tracker/Request.php @@ -20,6 +20,7 @@ use Matomo\Network\IPUtils; use Piwik\Piwik; use Piwik\Plugins\CustomVariables\CustomVariables; use Piwik\Plugins\UsersManager\UsersManager; +use Piwik\ProxyHttp; use Piwik\Tracker; use Piwik\Cache as PiwikCache; @@ -686,7 +687,12 @@ class Request $cookie = $this->makeThirdPartyCookieUID(); $idVisitor = bin2hex($idVisitor); $cookie->set(0, $idVisitor); - $cookie->save('None'); + if (ProxyHttp::isHttps()) { + $cookie->setSecure(true); + $cookie->save('None'); + } else { + $cookie->save('Lax'); + } Common::printDebug(sprintf("We set the visitor ID to %s in the 3rd party cookie...", $idVisitor)); } diff --git a/core/Tracker/Visit.php b/core/Tracker/Visit.php index a2ebfccec8..cd72105076 100644 --- a/core/Tracker/Visit.php +++ b/core/Tracker/Visit.php @@ -575,10 +575,10 @@ class Visit implements VisitInterface $date = Date::factory((int)$time, $timezone); // $date->isToday() is buggy when server and website timezones don't match - so we'll do our own checking - $startOfTomorrow = Date::factoryInTimezone('today', $timezone)->addDay(1); - $isLaterThanToday = $date->getTimestamp() >= $startOfTomorrow->getTimestamp(); - if ($isLaterThanToday) { - return; + $startOfToday = Date::factoryInTimezone('yesterday', $timezone)->addDay(1); + $isLaterThanYesterday = $date->getTimestamp() >= $startOfToday->getTimestamp(); + if ($isLaterThanYesterday) { + return; // don't try to invalidate archives for today or later } $this->invalidator->rememberToInvalidateArchivedReportsLater($idSite, $date); diff --git a/core/Twig.php b/core/Twig.php index 4a9200c691..165747e938 100644 --- a/core/Twig.php +++ b/core/Twig.php @@ -362,7 +362,7 @@ class Twig if (!empty($options['raw'])) { $template .= piwik_fix_lbrace($message); } else { - $template .= twig_escape_filter($twigEnv, $message, 'html'); + $template .= piwik_escape_filter($twigEnv, $message, 'html'); } $template .= '</div>'; diff --git a/core/Updates/3.13.4-b1.php b/core/Updates/3.13.4-b1.php new file mode 100644 index 0000000000..b56a322556 --- /dev/null +++ b/core/Updates/3.13.4-b1.php @@ -0,0 +1,23 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Updates; + +use Piwik\Plugins\Installation\ServerFilesGenerator; +use Piwik\Updater; +use Piwik\Updates as PiwikUpdates; + +class Updates_3_13_4_b1 extends PiwikUpdates +{ + public function doUpdate(Updater $updater) + { + // Fix issue with HeatmapSessionRecording on IIS (https://github.com/matomo-org/matomo/issues/15651) + ServerFilesGenerator::createFilesForSecurity(); + } +} diff --git a/core/View.php b/core/View.php index 14d8dd166d..a35903e3da 100644 --- a/core/View.php +++ b/core/View.php @@ -296,6 +296,9 @@ class View implements ViewInterface // don't send Referer-Header for outgoing links if (!empty($this->useStrictReferrerPolicy)) { Common::sendHeader('Referrer-Policy: same-origin'); + } else { + // always send explicit default header + Common::sendHeader('Referrer-Policy: no-referrer-when-downgrade'); } } diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index d94979c9c3..1e3933f972 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -9,11 +9,14 @@ namespace Piwik\Plugins\CoreAdminHome; use Piwik\API\Request; +use Piwik\Archive; +use Piwik\Archive\ArchiveInvalidator; use Piwik\ArchiveProcessor\Rules; use Piwik\Archive\ArchivePurger; use Piwik\Common; use Piwik\Config; use Piwik\Container\StaticContainer; +use Piwik\CronArchive; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; use Piwik\Db; @@ -60,6 +63,10 @@ class Tasks extends \Piwik\Plugin\Tasks public function schedule() { + // for browser triggered archiving, make sure we invalidate archives once a day just to make + // sure all archives that need to be invalidated get invalidated + $this->daily('invalidateOutdatedArchives', null, self::HIGH_PRIORITY); + // general data purge on older archive tables, executed daily $this->daily('purgeOutdatedArchives', null, self::HIGH_PRIORITY); @@ -81,6 +88,17 @@ class Tasks extends \Piwik\Plugin\Tasks $this->scheduleTrackingCodeReminderChecks(); } + public function invalidateOutdatedArchives() + { + if (!Rules::isBrowserTriggerEnabled()) { + $this->logger->info("Browser triggered archiving disabled, archives will be invalidated during core:archive."); + return; + } + + $cronArchive = new CronArchive(); + $cronArchive->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + } + private function scheduleTrackingCodeReminderChecks() { $daysToTrackedVisitsCheck = (int) Config::getInstance()->General['num_days_before_tracking_code_reminder']; diff --git a/plugins/CoreAdminHome/templates/optOut.twig b/plugins/CoreAdminHome/templates/optOut.twig index e97389b281..957a2c72cd 100644 --- a/plugins/CoreAdminHome/templates/optOut.twig +++ b/plugins/CoreAdminHome/templates/optOut.twig @@ -33,7 +33,14 @@ #} {% if showConfirmOnly %} <p>{{ 'CoreAdminHome_OptingYouOut'|translate }}</p> - <script>window.close();</script> + <script> + {# try to update nonce in iframe, so sending it a second time works #} + try { + window.opener.document.querySelector('[name="nonce"]').value = '{{ nonce }}'; + window.opener.document.querySelector('form').action = window.opener.document.querySelector('form').action.replace(/nonce=[0-9a-z]+/, 'nonce={{ nonce }}'); + } catch (e) {} + window.close(); + </script> <noscript> {% endif %} diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php index 431ab833f9..7449222454 100644 --- a/plugins/CoreAdminHome/tests/Integration/TasksTest.php +++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php @@ -132,6 +132,7 @@ class TasksTest extends IntegrationTestCase $tasks = array_map(function (Task $task) { return $task->getMethodName() . '.' . $task->getMethodParameter(); }, $tasks); $expected = [ + 'invalidateOutdatedArchives.', 'purgeOutdatedArchives.', 'purgeInvalidatedArchives.', 'purgeOrphanedArchives.', diff --git a/plugins/CoreHome/CoreHome.php b/plugins/CoreHome/CoreHome.php index 6587e5ad13..84d5fc66a0 100644 --- a/plugins/CoreHome/CoreHome.php +++ b/plugins/CoreHome/CoreHome.php @@ -11,13 +11,16 @@ namespace Piwik\Plugins\CoreHome; use Piwik\Archive\ArchiveInvalidator; use Piwik\Columns\ComputedMetricFactory; use Piwik\Columns\MetricsList; +use Piwik\Common; use Piwik\Container\StaticContainer; +use Piwik\DbHelper; use Piwik\IP; use Piwik\Piwik; use Piwik\Plugin\ArchivedMetric; use Piwik\Plugin\ComputedMetric; use Piwik\Plugin\ThemeStyles; use Piwik\SettingsServer; +use Piwik\Tracker\Model as TrackerModel; /** * @@ -59,6 +62,9 @@ class CoreHome extends \Piwik\Plugin /** @var ArchiveInvalidator $archiveInvalidator */ $archiveInvalidator = StaticContainer::get(ArchiveInvalidator::class); $cacheGeneral[ArchiveInvalidator::TRACKER_CACHE_KEY] = $archiveInvalidator->getAllRememberToInvalidateArchivedReportsLater(); + + $hasIndex = DbHelper::tableHasIndex(Common::prefixTable('log_visit'), 'index_idsite_idvisitor'); + $cacheGeneral[TrackerModel::CACHE_KEY_INDEX_IDSITE_IDVISITOR] = $hasIndex; } public function addStylesheets(&$mergedContent) diff --git a/plugins/CustomDimensions b/plugins/CustomDimensions -Subproject e0b449f67d8ccbf43ea25caa350a4dbe57c2389 +Subproject bdc988a16f5b604a3f882c366585467eb1bd54e diff --git a/plugins/Dashboard/Dashboard.php b/plugins/Dashboard/Dashboard.php index 069e6cb969..9a5e381513 100644 --- a/plugins/Dashboard/Dashboard.php +++ b/plugins/Dashboard/Dashboard.php @@ -157,11 +157,7 @@ class Dashboard extends \Piwik\Plugin if ($advertising->areAdsForProfessionalServicesEnabled() && $pluginManager->isPluginActivated('ProfessionalServices')) { $advertisingWidget = '{"uniqueId":"widgetProfessionalServicespromoServices","parameters":{"module":"ProfessionalServices","action":"promoServices"}},'; } - if (Piwik::hasUserSuperUserAccess()) { - $piwikPromoWidget = '{"uniqueId":"widgetCoreHomegetDonateForm","parameters":{"module":"CoreHome","action":"getDonateForm"}}'; - } else { - $piwikPromoWidget = '{"uniqueId":"widgetCoreHomegetPromoVideo","parameters":{"module":"CoreHome","action":"getPromoVideo"}}'; - } + $piwikPromoWidget = '{"uniqueId":"widgetCoreHomegetPromoVideo","parameters":{"module":"CoreHome","action":"getPromoVideo"}}'; $insightsWidget = ''; if ($pluginManager->isPluginActivated('Insights')) { $insightsWidget = '{"uniqueId":"widgetInsightsgetOverallMoversAndShakers","parameters":{"module":"Insights","action":"getOverallMoversAndShakers"}},'; diff --git a/plugins/Dashboard/tests/UI/expected-screenshots/DashboardManager_create_new.png b/plugins/Dashboard/tests/UI/expected-screenshots/DashboardManager_create_new.png index 476dd842f2..e7e03cfe7b 100644 --- a/plugins/Dashboard/tests/UI/expected-screenshots/DashboardManager_create_new.png +++ b/plugins/Dashboard/tests/UI/expected-screenshots/DashboardManager_create_new.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:3fb778b95d00d90336df9679e96014f513ff058df9f76ea502d023a757bf1285 -size 305893 +oid sha256:ceba2add6b8d6537ed128ad8930a48e10fdf701551bf6d51f98aaa60d91fd705 +size 317218 diff --git a/plugins/Dashboard/tests/UI/expected-screenshots/Dashboard_reset.png b/plugins/Dashboard/tests/UI/expected-screenshots/Dashboard_reset.png index dffc459de3..5e1d8961c9 100644 --- a/plugins/Dashboard/tests/UI/expected-screenshots/Dashboard_reset.png +++ b/plugins/Dashboard/tests/UI/expected-screenshots/Dashboard_reset.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:6718ef832b90b5fc4334fdd9f71769931495b0bdcac16a97a726e6cccf64052d -size 350683 +oid sha256:c826017ba202846f96d3b6c995d8d3c525d635975afec436baff38da7a8488e3 +size 378817 diff --git a/plugins/DeviceDetectorCache b/plugins/DeviceDetectorCache -Subproject a50e94149170ef73928184ba3f58ef2d8d1adb4 +Subproject a34a70f5d7db59745c5f64e07957846148b7a2f diff --git a/plugins/ExampleLogTables/tests/Fixtures/VisitsWithUserIdAndCustomData.php b/plugins/ExampleLogTables/tests/Fixtures/VisitsWithUserIdAndCustomData.php index 22ea04120e..f5e0f742ef 100644 --- a/plugins/ExampleLogTables/tests/Fixtures/VisitsWithUserIdAndCustomData.php +++ b/plugins/ExampleLogTables/tests/Fixtures/VisitsWithUserIdAndCustomData.php @@ -44,9 +44,10 @@ class VisitsWithUserIdAndCustomData extends Fixture foreach (array('user1', 'user2', 'user3', 'user4', false) as $key => $userId) { for ($numVisits = 0; $numVisits < ($key+1) * 10; $numVisits++) { + $visitDateTime = Date::factory($this->dateTime)->addHour($numVisits)->getDatetime(); + $t->setForceVisitDateTime($visitDateTime); $t->setUserId($userId); - $t->setPlugins($numVisits % 3 == 0, $numVisits % 5 == 0, $numVisits % 7 == 0); - $t->setBrowserHasCookies($numVisits % 3 == 0); + $t->setVisitorId(str_pad($numVisits.$key, 16, 'a')); $t->setCountry(self::$countryCodes[$numVisits % count(self::$countryCodes)]); if ($numVisits % 5 == 0) { @@ -72,7 +73,7 @@ class VisitsWithUserIdAndCustomData extends Fixture $t->setForceNewVisit(); $t->setUrl('http://example.org/my/dir/page' . ($numVisits % 4)); - $visitDateTime = Date::factory($this->dateTime)->addHour($numVisits*6)->getDatetime(); + $visitDateTime = Date::factory($this->dateTime)->addHour($numVisits+6)->getDatetime(); $t->setForceVisitDateTime($visitDateTime); if ($numVisits % 7 == 0) { @@ -82,7 +83,7 @@ class VisitsWithUserIdAndCustomData extends Fixture self::assertTrue($t->doTrackPageView('incredible title ' . ($numVisits % 3))); if ($numVisits % 9 == 0) { - $t->setForceVisitDateTime(Date::factory($this->dateTime)->addHour($numVisits*6.1)->getDatetime()); + $t->setForceVisitDateTime(Date::factory($this->dateTime)->addHour($numVisits+6.1)->getDatetime()); $t->addEcommerceItem('SKU VERY nice indeed ' . ($numVisits%3), 'PRODUCT name ' . ($numVisits%4), 'category ' . ($numVisits%5), $numVisits*2.79); self::assertTrue($t->doTrackEcommerceCartUpdate($numVisits*17)); } diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__Actions.get_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__Actions.get_month.xml index d73153a828..61570f95fd 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__Actions.get_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__Actions.get_month.xml @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="utf-8" ?> <result> <nb_pageviews>45</nb_pageviews> - <nb_uniq_pageviews>40</nb_uniq_pageviews> + <nb_uniq_pageviews>45</nb_uniq_pageviews> <nb_downloads>7</nb_downloads> <nb_uniq_downloads>7</nb_uniq_downloads> <nb_outlinks>0</nb_outlinks> diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__UserId.getUsers_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__UserId.getUsers_month.xml index e9d5ed7b27..66ddc67d77 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__UserId.getUsers_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__UserId.getUsers_month.xml @@ -2,26 +2,26 @@ <result> <row> <label>user3</label> - <nb_visits>34</nb_visits> + <nb_visits>33</nb_visits> <nb_actions>53</nb_actions> - <max_actions>3</max_actions> - <sum_visit_length>20</sum_visit_length> - <bounce_count>19</bounce_count> + <max_actions>4</max_actions> + <sum_visit_length>741</sum_visit_length> + <bounce_count>16</bounce_count> <nb_visits_converted>0</nb_visits_converted> - <sum_daily_nb_uniq_visitors>8</sum_daily_nb_uniq_visitors> - <sum_daily_nb_users>8</sum_daily_nb_users> + <sum_daily_nb_uniq_visitors>30</sum_daily_nb_uniq_visitors> + <sum_daily_nb_users>2</sum_daily_nb_users> </row> <row> <label>user1</label> - <nb_visits>12</nb_visits> + <nb_visits>13</nb_visits> <nb_actions>18</nb_actions> - <max_actions>3</max_actions> - <sum_visit_length>6</sum_visit_length> - <bounce_count>7</bounce_count> + <max_actions>2</max_actions> + <sum_visit_length>727</sum_visit_length> + <bounce_count>8</bounce_count> <nb_visits_converted>0</nb_visits_converted> - <sum_daily_nb_uniq_visitors>3</sum_daily_nb_uniq_visitors> - <sum_daily_nb_users>3</sum_daily_nb_users> + <sum_daily_nb_uniq_visitors>10</sum_daily_nb_uniq_visitors> + <sum_daily_nb_users>2</sum_daily_nb_users> </row> </result>
\ No newline at end of file diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__VisitsSummary.get_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__VisitsSummary.get_month.xml index ff76df488f..94efd54469 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__VisitsSummary.get_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_admin__VisitsSummary.get_month.xml @@ -1,14 +1,14 @@ <?xml version="1.0" encoding="utf-8" ?> <result> - <nb_uniq_visitors>1</nb_uniq_visitors> + <nb_uniq_visitors>40</nb_uniq_visitors> <nb_users>2</nb_users> <nb_visits>46</nb_visits> <nb_actions>71</nb_actions> <nb_visits_converted>0</nb_visits_converted> - <bounce_count>26</bounce_count> - <sum_visit_length>26</sum_visit_length> - <max_actions>3</max_actions> - <bounce_rate>57%</bounce_rate> + <bounce_count>24</bounce_count> + <sum_visit_length>1468</sum_visit_length> + <max_actions>4</max_actions> + <bounce_rate>52%</bounce_rate> <nb_actions_per_visit>1.5</nb_actions_per_visit> - <avg_time_on_site>1</avg_time_on_site> + <avg_time_on_site>32</avg_time_on_site> </result>
\ No newline at end of file diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__Actions.get_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__Actions.get_month.xml index 189d44ca43..2d7edd1635 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__Actions.get_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__Actions.get_month.xml @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="utf-8" ?> <result> <nb_pageviews>171</nb_pageviews> - <nb_uniq_pageviews>151</nb_uniq_pageviews> + <nb_uniq_pageviews>171</nb_uniq_pageviews> <nb_downloads>24</nb_downloads> <nb_uniq_downloads>24</nb_uniq_downloads> <nb_outlinks>0</nb_outlinks> diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__UserId.getUsers_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__UserId.getUsers_month.xml index 1e4420b714..88cc874456 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__UserId.getUsers_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__UserId.getUsers_month.xml @@ -2,50 +2,50 @@ <result> <row> <label>user4</label> - <nb_visits>45</nb_visits> - <nb_actions>74</nb_actions> - <max_actions>3</max_actions> - <sum_visit_length>28</sum_visit_length> - <bounce_count>22</bounce_count> + <nb_visits>43</nb_visits> + <nb_actions>70</nb_actions> + <max_actions>4</max_actions> + <sum_visit_length>1108</sum_visit_length> + <bounce_count>20</bounce_count> <nb_visits_converted>0</nb_visits_converted> - <sum_daily_nb_uniq_visitors>11</sum_daily_nb_uniq_visitors> - <sum_daily_nb_users>11</sum_daily_nb_users> + <sum_daily_nb_uniq_visitors>40</sum_daily_nb_uniq_visitors> + <sum_daily_nb_users>3</sum_daily_nb_users> </row> <row> <label>user3</label> - <nb_visits>34</nb_visits> + <nb_visits>33</nb_visits> <nb_actions>53</nb_actions> - <max_actions>3</max_actions> - <sum_visit_length>20</sum_visit_length> - <bounce_count>19</bounce_count> + <max_actions>4</max_actions> + <sum_visit_length>741</sum_visit_length> + <bounce_count>16</bounce_count> <nb_visits_converted>0</nb_visits_converted> - <sum_daily_nb_uniq_visitors>8</sum_daily_nb_uniq_visitors> - <sum_daily_nb_users>8</sum_daily_nb_users> + <sum_daily_nb_uniq_visitors>30</sum_daily_nb_uniq_visitors> + <sum_daily_nb_users>2</sum_daily_nb_users> </row> <row> <label>user2</label> - <nb_visits>22</nb_visits> + <nb_visits>23</nb_visits> <nb_actions>35</nb_actions> - <max_actions>3</max_actions> - <sum_visit_length>12</sum_visit_length> - <bounce_count>12</bounce_count> + <max_actions>2</max_actions> + <sum_visit_length>735</sum_visit_length> + <bounce_count>11</bounce_count> <nb_visits_converted>0</nb_visits_converted> - <sum_daily_nb_uniq_visitors>6</sum_daily_nb_uniq_visitors> - <sum_daily_nb_users>6</sum_daily_nb_users> + <sum_daily_nb_uniq_visitors>20</sum_daily_nb_uniq_visitors> + <sum_daily_nb_users>2</sum_daily_nb_users> </row> <row> <label>user1</label> - <nb_visits>12</nb_visits> + <nb_visits>13</nb_visits> <nb_actions>18</nb_actions> - <max_actions>3</max_actions> - <sum_visit_length>6</sum_visit_length> - <bounce_count>7</bounce_count> + <max_actions>2</max_actions> + <sum_visit_length>727</sum_visit_length> + <bounce_count>8</bounce_count> <nb_visits_converted>0</nb_visits_converted> - <sum_daily_nb_uniq_visitors>3</sum_daily_nb_uniq_visitors> - <sum_daily_nb_users>3</sum_daily_nb_users> + <sum_daily_nb_uniq_visitors>10</sum_daily_nb_uniq_visitors> + <sum_daily_nb_users>2</sum_daily_nb_users> </row> </result>
\ No newline at end of file diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__VisitsSummary.get_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__VisitsSummary.get_month.xml index 66190100e5..23b293cf53 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__VisitsSummary.get_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_all__VisitsSummary.get_month.xml @@ -1,14 +1,14 @@ <?xml version="1.0" encoding="utf-8" ?> <result> - <nb_uniq_visitors>1</nb_uniq_visitors> + <nb_uniq_visitors>150</nb_uniq_visitors> <nb_users>4</nb_users> - <nb_visits>164</nb_visits> + <nb_visits>191</nb_visits> <nb_actions>264</nb_actions> <nb_visits_converted>0</nb_visits_converted> - <bounce_count>86</bounce_count> - <sum_visit_length>97</sum_visit_length> - <max_actions>3</max_actions> - <bounce_rate>52%</bounce_rate> - <nb_actions_per_visit>1.6</nb_actions_per_visit> - <avg_time_on_site>1</avg_time_on_site> + <bounce_count>120</bounce_count> + <sum_visit_length>5492</sum_visit_length> + <max_actions>4</max_actions> + <bounce_rate>63%</bounce_rate> + <nb_actions_per_visit>1.4</nb_actions_per_visit> + <avg_time_on_site>29</avg_time_on_site> </result>
\ No newline at end of file diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__Actions.get_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__Actions.get_month.xml index c693c3bcc9..d6b21a8339 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__Actions.get_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__Actions.get_month.xml @@ -1,11 +1,11 @@ <?xml version="1.0" encoding="utf-8" ?> <result> <nb_pageviews>57</nb_pageviews> - <nb_uniq_pageviews>50</nb_uniq_pageviews> + <nb_uniq_pageviews>57</nb_uniq_pageviews> <nb_downloads>8</nb_downloads> <nb_uniq_downloads>8</nb_uniq_downloads> <nb_outlinks>0</nb_outlinks> <nb_uniq_outlinks>0</nb_uniq_outlinks> - <nb_searches>12</nb_searches> + <nb_searches>10</nb_searches> <nb_keywords>8</nb_keywords> </result>
\ No newline at end of file diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__UserId.getUsers_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__UserId.getUsers_month.xml index 06f94bf4e8..9f798cd7bf 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__UserId.getUsers_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__UserId.getUsers_month.xml @@ -2,26 +2,26 @@ <result> <row> <label>user4</label> - <nb_visits>45</nb_visits> - <nb_actions>74</nb_actions> - <max_actions>3</max_actions> - <sum_visit_length>28</sum_visit_length> - <bounce_count>22</bounce_count> + <nb_visits>43</nb_visits> + <nb_actions>70</nb_actions> + <max_actions>4</max_actions> + <sum_visit_length>1108</sum_visit_length> + <bounce_count>20</bounce_count> <nb_visits_converted>0</nb_visits_converted> - <sum_daily_nb_uniq_visitors>11</sum_daily_nb_uniq_visitors> - <sum_daily_nb_users>11</sum_daily_nb_users> + <sum_daily_nb_uniq_visitors>40</sum_daily_nb_uniq_visitors> + <sum_daily_nb_users>3</sum_daily_nb_users> </row> <row> <label>user1</label> - <nb_visits>12</nb_visits> + <nb_visits>13</nb_visits> <nb_actions>18</nb_actions> - <max_actions>3</max_actions> - <sum_visit_length>6</sum_visit_length> - <bounce_count>7</bounce_count> + <max_actions>2</max_actions> + <sum_visit_length>727</sum_visit_length> + <bounce_count>8</bounce_count> <nb_visits_converted>0</nb_visits_converted> - <sum_daily_nb_uniq_visitors>3</sum_daily_nb_uniq_visitors> - <sum_daily_nb_users>3</sum_daily_nb_users> + <sum_daily_nb_uniq_visitors>10</sum_daily_nb_uniq_visitors> + <sum_daily_nb_users>2</sum_daily_nb_users> </row> </result>
\ No newline at end of file diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__VisitsSummary.get_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__VisitsSummary.get_month.xml index a8c9a77cac..0dec751572 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__VisitsSummary.get_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_men__VisitsSummary.get_month.xml @@ -1,14 +1,14 @@ <?xml version="1.0" encoding="utf-8" ?> <result> - <nb_uniq_visitors>1</nb_uniq_visitors> + <nb_uniq_visitors>50</nb_uniq_visitors> <nb_users>2</nb_users> - <nb_visits>57</nb_visits> - <nb_actions>92</nb_actions> + <nb_visits>56</nb_visits> + <nb_actions>88</nb_actions> <nb_visits_converted>0</nb_visits_converted> - <bounce_count>29</bounce_count> - <sum_visit_length>34</sum_visit_length> - <max_actions>3</max_actions> - <bounce_rate>51%</bounce_rate> + <bounce_count>28</bounce_count> + <sum_visit_length>1835</sum_visit_length> + <max_actions>4</max_actions> + <bounce_rate>50%</bounce_rate> <nb_actions_per_visit>1.6</nb_actions_per_visit> - <avg_time_on_site>1</avg_time_on_site> + <avg_time_on_site>33</avg_time_on_site> </result>
\ No newline at end of file diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__Actions.get_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__Actions.get_month.xml index 4269b53424..06508b8e7a 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__Actions.get_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__Actions.get_month.xml @@ -1,7 +1,7 @@ <?xml version="1.0" encoding="utf-8" ?> <result> <nb_pageviews>57</nb_pageviews> - <nb_uniq_pageviews>50</nb_uniq_pageviews> + <nb_uniq_pageviews>57</nb_uniq_pageviews> <nb_downloads>8</nb_downloads> <nb_uniq_downloads>8</nb_uniq_downloads> <nb_outlinks>0</nb_outlinks> diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__UserId.getUsers_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__UserId.getUsers_month.xml index d9abec4587..2fe5964a40 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__UserId.getUsers_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__UserId.getUsers_month.xml @@ -2,26 +2,26 @@ <result> <row> <label>user3</label> - <nb_visits>34</nb_visits> + <nb_visits>33</nb_visits> <nb_actions>53</nb_actions> - <max_actions>3</max_actions> - <sum_visit_length>20</sum_visit_length> - <bounce_count>19</bounce_count> + <max_actions>4</max_actions> + <sum_visit_length>741</sum_visit_length> + <bounce_count>16</bounce_count> <nb_visits_converted>0</nb_visits_converted> - <sum_daily_nb_uniq_visitors>8</sum_daily_nb_uniq_visitors> - <sum_daily_nb_users>8</sum_daily_nb_users> + <sum_daily_nb_uniq_visitors>30</sum_daily_nb_uniq_visitors> + <sum_daily_nb_users>2</sum_daily_nb_users> </row> <row> <label>user2</label> - <nb_visits>22</nb_visits> + <nb_visits>23</nb_visits> <nb_actions>35</nb_actions> - <max_actions>3</max_actions> - <sum_visit_length>12</sum_visit_length> - <bounce_count>12</bounce_count> + <max_actions>2</max_actions> + <sum_visit_length>735</sum_visit_length> + <bounce_count>11</bounce_count> <nb_visits_converted>0</nb_visits_converted> - <sum_daily_nb_uniq_visitors>6</sum_daily_nb_uniq_visitors> - <sum_daily_nb_users>6</sum_daily_nb_users> + <sum_daily_nb_uniq_visitors>20</sum_daily_nb_uniq_visitors> + <sum_daily_nb_users>2</sum_daily_nb_users> </row> </result>
\ No newline at end of file diff --git a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__VisitsSummary.get_month.xml b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__VisitsSummary.get_month.xml index 7011733c70..22d38cc245 100644 --- a/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__VisitsSummary.get_month.xml +++ b/plugins/ExampleLogTables/tests/System/expected/test_ExampleLogTables_women__VisitsSummary.get_month.xml @@ -1,14 +1,14 @@ <?xml version="1.0" encoding="utf-8" ?> <result> - <nb_uniq_visitors>1</nb_uniq_visitors> + <nb_uniq_visitors>50</nb_uniq_visitors> <nb_users>2</nb_users> <nb_visits>56</nb_visits> <nb_actions>88</nb_actions> <nb_visits_converted>0</nb_visits_converted> - <bounce_count>31</bounce_count> - <sum_visit_length>32</sum_visit_length> - <max_actions>3</max_actions> - <bounce_rate>55%</bounce_rate> + <bounce_count>27</bounce_count> + <sum_visit_length>1476</sum_visit_length> + <max_actions>4</max_actions> + <bounce_rate>48%</bounce_rate> <nb_actions_per_visit>1.6</nb_actions_per_visit> - <avg_time_on_site>1</avg_time_on_site> + <avg_time_on_site>26</avg_time_on_site> </result>
\ No newline at end of file diff --git a/plugins/Goals/Controller.php b/plugins/Goals/Controller.php index 9da9138fa3..3367fcaf93 100644 --- a/plugins/Goals/Controller.php +++ b/plugins/Goals/Controller.php @@ -24,17 +24,12 @@ use Piwik\View; */ class Controller extends \Piwik\Plugin\Controller { - const CONVERSION_RATE_PRECISION = 1; - /** * Number of "Your top converting keywords/etc are" to display in the per Goal overview page * @var int */ const COUNT_TOP_ROWS_TO_DISPLAY = 3; - const ECOMMERCE_LOG_SHOW_ORDERS = 1; - const ECOMMERCE_LOG_SHOW_ABANDONED_CARTS = 2; - protected $goalColumnNameToLabel = array( 'avg_order_revenue' => 'General_AverageOrderValue', 'nb_conversions' => 'Goals_ColumnConversions', @@ -59,10 +54,6 @@ class Controller extends \Piwik\Plugin\Controller } } - if (!is_numeric($conversionRate)) { - $conversionRate = sprintf('%.' . self::CONVERSION_RATE_PRECISION . 'f%%', $conversionRate); - } - return $conversionRate; } diff --git a/plugins/Installation/ServerFilesGenerator.php b/plugins/Installation/ServerFilesGenerator.php index 3bc5750e22..7ed67db57a 100644 --- a/plugins/Installation/ServerFilesGenerator.php +++ b/plugins/Installation/ServerFilesGenerator.php @@ -167,6 +167,12 @@ Header set Cache-Control \"Cache-Control: private, no-cache, no-store\" '/vendor', '/plugins', ); + + $additionForPlugins = ' + <alwaysAllowedUrls> + <add url="/plugins/HeatmapSessionRecording/configs.php" /> + </alwaysAllowedUrls>'; + foreach ($directoriesToProtect as $directoryToProtect) { @file_put_contents(PIWIK_INCLUDE_PATH . $directoryToProtect . '/web.config', '<?xml version="1.0" encoding="UTF-8"?> @@ -176,7 +182,7 @@ Header set Cache-Control \"Cache-Control: private, no-cache, no-store\" <requestFiltering> <denyUrlSequences> <add sequence=".php" /> - </denyUrlSequences> + </denyUrlSequences>' . ($directoryToProtect === '/plugins' ? $additionForPlugins : '') . ' </requestFiltering> </security> </system.webServer> diff --git a/plugins/Live/Model.php b/plugins/Live/Model.php index 89c32d2b12..fe167fcf33 100644 --- a/plugins/Live/Model.php +++ b/plugins/Live/Model.php @@ -16,6 +16,7 @@ use Piwik\Config; use Piwik\Container\StaticContainer; use Piwik\Date; use Piwik\Db; +use Piwik\DbHelper; use Piwik\Period; use Piwik\Period\Range; use Piwik\Piwik; @@ -121,7 +122,7 @@ class Model try { $visits = $readerDb->fetchAll($sql, $bind); } catch (Exception $e) { - $this->handleMaxExecutionTimeError($readerDb, $e, $sql, $bind, $segment, $dateStart, $dateEnd, $minTimestamp, $limit); + $this->handleMaxExecutionTimeError($readerDb, $e, $segment, $dateStart, $dateEnd, $minTimestamp, $limit, ['sql' => $sql, 'bind' => $bind,]); throw $e; } return $visits; @@ -130,17 +131,16 @@ class Model /** * @param \Piwik\Tracker\Db|\Piwik\Db\AdapterInterface|\Piwik\Db $readerDb * @param Exception $e - * @param $sql - * @param array $bind * @param $segment * @param $dateStart * @param $dateEnd * @param $minTimestamp * @param $limit + * @param $parameters * * @throws MaxExecutionTimeExceededException */ - public function handleMaxExecutionTimeError($readerDb, $e, $sql, $bind, $segment, $dateStart, $dateEnd, $minTimestamp, $limit) + public static function handleMaxExecutionTimeError($readerDb, $e, $segment, $dateStart, $dateEnd, $minTimestamp, $limit, $parameters) { // we also need to check for the 'maximum statement execution time exceeded' text as the query might be // aborted at different stages and we can't really know all the possible codes at which it may be aborted etc @@ -148,39 +148,41 @@ class Model || $readerDb->isErrNo($e, DbMigration::ERROR_CODE_MAX_EXECUTION_TIME_EXCEEDED_SORT_ABORTED) || strpos($e->getMessage(), 'maximum statement execution time exceeded') !== false; - if ($isMaxExecutionTimeError) { - $message = ''; + if (false === $isMaxExecutionTimeError) { + return; + } + + $message = ''; - if ($this->isLookingAtMoreThanOneDay($dateStart, $dateEnd, $minTimestamp)) { - $message .= ' ' . Piwik::translate('Live_QueryMaxExecutionTimeExceededReasonDateRange'); - } + if (self::isLookingAtMoreThanOneDay($dateStart, $dateEnd, $minTimestamp)) { + $message .= ' ' . Piwik::translate('Live_QueryMaxExecutionTimeExceededReasonDateRange'); + } - if (!empty($segment)) { - $message .= ' ' . Piwik::translate('Live_QueryMaxExecutionTimeExceededReasonSegment'); - } + if (!empty($segment)) { + $message .= ' ' . Piwik::translate('Live_QueryMaxExecutionTimeExceededReasonSegment'); + } - $limitThatCannotBeSelectedInUiButOnlyApi = 550; - if ($limit > $limitThatCannotBeSelectedInUiButOnlyApi) { - $message .= ' ' . Piwik::translate('Live_QueryMaxExecutionTimeExceededLimit'); - } + $limitThatCannotBeSelectedInUiButOnlyApi = 550; + if ($limit > $limitThatCannotBeSelectedInUiButOnlyApi) { + $message .= ' ' . Piwik::translate('Live_QueryMaxExecutionTimeExceededLimit'); + } - if (empty($message)) { - $message .= ' ' . Piwik::translate('Live_QueryMaxExecutionTimeExceededReasonUnknown'); - } + if (empty($message)) { + $message .= ' ' . Piwik::translate('Live_QueryMaxExecutionTimeExceededReasonUnknown'); + } - $message = Piwik::translate('Live_QueryMaxExecutionTimeExceeded') . ' ' . $message; + $message = Piwik::translate('Live_QueryMaxExecutionTimeExceeded') . ' ' . $message; - $params = array( - 'sql' => $sql, 'bind' => $bind, 'segment' => $segment, 'limit' => $limit - ); + $params = array_merge($parameters, [ + 'segment' => $segment, 'limit' => $limit + ]); - /** - * @ignore - * @internal - */ - Piwik::postEvent('Live.queryMaxExecutionTimeExceeded', array($params)); - throw new MaxExecutionTimeExceededException($message); - } + /** + * @ignore + * @internal + */ + Piwik::postEvent('Live.queryMaxExecutionTimeExceeded', array($params)); + throw new MaxExecutionTimeExceededException($message); } /** @@ -190,7 +192,7 @@ class Model * @return bool * @throws Exception */ - public function isLookingAtMoreThanOneDay($dateStart, $dateEnd, $minTimestamp) + public static function isLookingAtMoreThanOneDay($dateStart, $dateEnd, $minTimestamp) { if (!$dateStart) { if (!$minTimestamp) { @@ -493,35 +495,14 @@ class Model $bind = $innerQuery['bind']; - $maxExecutionTimeHint = $this->getMaxExecutionTimeMySQLHint(); - if ($visitorId) { + if (!$visitorId) { // for now let's not apply when looking for a specific visitor - $maxExecutionTimeHint = ''; - } - if ($maxExecutionTimeHint) { - $innerQuery['sql'] = trim($innerQuery['sql']); - $pos = stripos($innerQuery['sql'], 'SELECT'); - if ($pos !== false) { - $innerQuery['sql'] = substr_replace($innerQuery['sql'], 'SELECT ' . $maxExecutionTimeHint, $pos, strlen('SELECT')); - } + $innerQuery['sql'] = DbHelper::addMaxExecutionTimeHintToQuery($innerQuery['sql'], Config::getInstance()->General['live_query_max_execution_time']); } return array($innerQuery['sql'], $bind); } - private function getMaxExecutionTimeMySQLHint() - { - $general = Config::getInstance()->General; - $maxExecutionTime = $general['live_query_max_execution_time']; - $maxExecutionTimeHint = ''; - if (is_numeric($maxExecutionTime) && $maxExecutionTime > 0) { - $timeInMs = $maxExecutionTime * 1000; - $timeInMs = (int) $timeInMs; - $maxExecutionTimeHint = ' /*+ MAX_EXECUTION_TIME('.$timeInMs.') */ '; - } - return $maxExecutionTimeHint; - } - /** * @param $idSite * @return Site diff --git a/plugins/Live/tests/Integration/ModelTest.php b/plugins/Live/tests/Integration/ModelTest.php index ea2b104732..d66872e44d 100644 --- a/plugins/Live/tests/Integration/ModelTest.php +++ b/plugins/Live/tests/Integration/ModelTest.php @@ -82,8 +82,7 @@ class ModelTest extends IntegrationTestCase $dateEnd = Date::now(); $minTimestamp = 1; $limit = 50; - $model = new Model(); - $model->handleMaxExecutionTimeError($db, $e, $sql, $bind, $segment, $dateStart, $dateEnd, $minTimestamp, $limit); + Model::handleMaxExecutionTimeError($db, $e, $segment, $dateStart, $dateEnd, $minTimestamp, $limit, [$sql, $bind]); $this->assertTrue(true); } @@ -101,8 +100,7 @@ class ModelTest extends IntegrationTestCase $dateEnd = Date::now(); $minTimestamp = null; $limit = 50; - $model = new Model(); - $model->handleMaxExecutionTimeError($db, $e, $sql, $bind, $segment, $dateStart, $dateEnd, $minTimestamp, $limit); + Model::handleMaxExecutionTimeError($db, $e, $segment, $dateStart, $dateEnd, $minTimestamp, $limit, [$sql, $bind]); } public function test_handleMaxExecutionTimeError_whenTimeIsExceeded_manyReasonsFound() @@ -112,15 +110,12 @@ class ModelTest extends IntegrationTestCase $db = Db::get(); $e = new \Exception('Query execution was interrupted, maximum statement execution time exceeded'); - $sql = 'SELECT 1'; - $bind = array(); $segment = 'userId>=1'; $dateStart = Date::now()->subDay(10); $dateEnd = Date::now(); $minTimestamp = null; $limit = 5000; - $model = new Model(); - $model->handleMaxExecutionTimeError($db, $e, $sql, $bind, $segment, $dateStart, $dateEnd, $minTimestamp, $limit); + Model::handleMaxExecutionTimeError($db, $e, $segment, $dateStart, $dateEnd, $minTimestamp, $limit, ['param' => 'value']); } public function test_getStandAndEndDate() diff --git a/plugins/Marketplace/lang/en.json b/plugins/Marketplace/lang/en.json index a84b879240..7d9cbf34aa 100644 --- a/plugins/Marketplace/lang/en.json +++ b/plugins/Marketplace/lang/en.json @@ -8,6 +8,7 @@ "AllowedUploadFormats": "You may upload a plugin or theme in .zip format via this page.", "Authors": "Authors", "Browse": "Browse", + "SupportMatomoThankYou": "Any purchase will help fund the future of the Matomo open-source project. Thank you for your support!", "LatestMarketplaceUpdates": "Latest Marketplace Updates", "BackToMarketplace": "Back to Marketplace", "BrowseMarketplace": "Browse Marketplace", diff --git a/plugins/Marketplace/templates/getPremiumFeatures.twig b/plugins/Marketplace/templates/getPremiumFeatures.twig index fa0fe83744..3663ff69c0 100644 --- a/plugins/Marketplace/templates/getPremiumFeatures.twig +++ b/plugins/Marketplace/templates/getPremiumFeatures.twig @@ -1,5 +1,8 @@ <div class="getNewPlugins getPremiumFeatures widgetBody"> <div class="row"> + <div class="col s12 m12"> + <h3 style="margin-bottom: 28px;">{{ 'Marketplace_SupportMatomoThankYou'|translate }}</h3></div> + {% for plugin in plugins %} <div class="col s12 m4"> diff --git a/plugins/Monolog/Processor/ExceptionToTextProcessor.php b/plugins/Monolog/Processor/ExceptionToTextProcessor.php index ab828dc820..0089b27d94 100644 --- a/plugins/Monolog/Processor/ExceptionToTextProcessor.php +++ b/plugins/Monolog/Processor/ExceptionToTextProcessor.php @@ -85,6 +85,10 @@ class ExceptionToTextProcessor public static function getWholeBacktrace(\Exception $exception, $shouldPrintBacktrace = true) { + if (!$shouldPrintBacktrace) { + return $exception->getMessage(); + } + $message = ""; $e = $exception; diff --git a/plugins/PrivacyManager/tests/Integration/DataPurgingTest.php b/plugins/PrivacyManager/tests/Integration/DataPurgingTest.php index 076cf9989a..9c1c95dde4 100644 --- a/plugins/PrivacyManager/tests/Integration/DataPurgingTest.php +++ b/plugins/PrivacyManager/tests/Integration/DataPurgingTest.php @@ -10,6 +10,7 @@ namespace Piwik\Plugins\PrivacyManager\tests\Integration; use Piwik\Archive; use Piwik\Common; use Piwik\Config; +use Piwik\Container\StaticContainer; use Piwik\DataAccess\RawLogDao; use Piwik\Date; use Piwik\Db; @@ -741,10 +742,14 @@ class DataPurgingTest extends IntegrationTestCase $range = $rangeStart->toString('Y-m-d') . "," . $rangeEnd->toString('Y-m-d'); $rangeArchive = Archive::build(self::$idSite, 'range', $range); - $rangeArchive->getNumeric('nb_visits', 'nb_hits'); + $rangeArchive->getNumeric(['nb_visits']); APIVisitorInterest::getInstance()->getNumberOfVisitsPerVisitDuration(self::$idSite, 'range', $range); + // remove invalidated + StaticContainer::get(Archive\ArchivePurger::class)->purgeInvalidatedArchivesFrom(Date::factory('2012-01-01')); + StaticContainer::get(Archive\ArchivePurger::class)->purgeInvalidatedArchivesFrom(Date::factory('2012-02-01')); + // when archiving is initiated, the archive metrics & reports for EVERY loaded plugin // are archived. don't want this test to depend on every possible metric, so get rid of // the unwanted archive data now. diff --git a/plugins/PrivacyManager/tests/Integration/Model/DataSubjectsTest.php b/plugins/PrivacyManager/tests/Integration/Model/DataSubjectsTest.php index 3cc563081f..27a8dbc52b 100644 --- a/plugins/PrivacyManager/tests/Integration/Model/DataSubjectsTest.php +++ b/plugins/PrivacyManager/tests/Integration/Model/DataSubjectsTest.php @@ -366,7 +366,7 @@ class DataSubjectsTest extends IntegrationTestCase $this->removeArchiveInvalidationOptions(); $visitDate = Date::factory($this->theFixture->dateTime); - $key = 'report_to_invalidate_' . $idSite . '_' . $visitDate->toString('Y-m-d') . '_12345'; + $key = '4444_report_to_invalidate_' . $idSite . '_' . $visitDate->toString('Y-m-d') . '_12345'; Option::set($key, '1'); $this->assertArchivesHaveBeenInvalidated($visitDate, $idSite); @@ -402,14 +402,14 @@ class DataSubjectsTest extends IntegrationTestCase private function assertArchivesHaveNotBeenInvalidated(Date $visitDate, $idSite) { $key = 'report_to_invalidate_' . $idSite . '_' . $visitDate->toString('Y-m-d'); - $value = Option::getLike($key . '%'); + $value = Option::getLike('%' . $key . '%'); $this->assertEmpty($value); } private function assertArchivesHaveBeenInvalidated(Date $visitDate, $idSite) { $key = 'report_to_invalidate_' . $idSite . '_' . $visitDate->toString('Y-m-d'); - $value = Option::getLike($key . '%'); + $value = Option::getLike('%' . $key . '%'); $this->assertNotEmpty($value); $this->assertEquals('1', array_values($value)[0]); } @@ -508,7 +508,7 @@ class DataSubjectsTest extends IntegrationTestCase private function removeArchiveInvalidationOptions() { - Option::deleteLike('report_to_invalidate_%'); + Option::deleteLike('%report_to_invalidate_%'); } private function setWebsiteTimezone($idSite, $timezone) diff --git a/plugins/Proxy/Controller.php b/plugins/Proxy/Controller.php index 0dfa3d03e3..4ffc45cc96 100644 --- a/plugins/Proxy/Controller.php +++ b/plugins/Proxy/Controller.php @@ -11,6 +11,7 @@ namespace Piwik\Plugins\Proxy; use Piwik\AssetManager; use Piwik\AssetManager\UIAsset; use Piwik\Common; +use Piwik\Exception\StylesheetLessCompileException; use Piwik\Piwik; use Piwik\ProxyHttp; use Piwik\Url; @@ -33,7 +34,11 @@ class Controller extends \Piwik\Plugin\Controller */ public function getCss() { - $cssMergedFile = AssetManager::getInstance()->getMergedStylesheet(); + try { + $cssMergedFile = AssetManager::getInstance()->getMergedStylesheet(); + } catch (StylesheetLessCompileException $exception) { + $cssMergedFile = AssetManager::getInstance()->getMergedStylesheet(); + } ProxyHttp::serverStaticFile($cssMergedFile->getAbsoluteLocation(), "text/css"); } diff --git a/plugins/SitesManager/tests/Integration/SitesManagerTest.php b/plugins/SitesManager/tests/Integration/SitesManagerTest.php index 24be827b95..51f07cdc2f 100644 --- a/plugins/SitesManager/tests/Integration/SitesManagerTest.php +++ b/plugins/SitesManager/tests/Integration/SitesManagerTest.php @@ -63,12 +63,14 @@ class SitesManagerTest extends IntegrationTestCase $archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-06')); $archive->rememberToInvalidateArchivedReportsLater(4949, Date::factory('2014-04-05')); - $expected = array( - '2014-04-05' => array($this->siteId, 4949), - '2014-04-06' => array($this->siteId) - ); + $remembered = $archive->getRememberedArchivedReportsThatShouldBeInvalidated(); + $this->assertCount(2, $remembered); - $this->assertEquals($expected, $archive->getRememberedArchivedReportsThatShouldBeInvalidated()); + sort($remembered['2014-04-05']); + $this->assertSame(array($this->siteId, 4949), $remembered['2014-04-05']); + + sort($remembered['2014-04-06']); + $this->assertSame(array($this->siteId), $remembered['2014-04-06']); $this->manager->onSiteDeleted($this->siteId); diff --git a/plugins/Transitions/API.php b/plugins/Transitions/API.php index 88bca868ef..b26b0bdc39 100644 --- a/plugins/Transitions/API.php +++ b/plugins/Transitions/API.php @@ -12,15 +12,18 @@ namespace Piwik\Plugins\Transitions; use Exception; use Piwik\ArchiveProcessor; use Piwik\Common; +use Piwik\Config; use Piwik\DataAccess\LogAggregator; use Piwik\DataArray; use Piwik\DataTable; use Piwik\DataTable\Row; +use Piwik\Db; use Piwik\Metrics; use Piwik\Period; use Piwik\Piwik; use Piwik\Plugins\Actions\Actions; use Piwik\Plugins\Actions\ArchivingHelper; +use Piwik\Plugins\Live\Model; use Piwik\RankingQuery; use Piwik\Segment; use Piwik\Segment\SegmentExpression; @@ -81,25 +84,40 @@ class API extends \Piwik\Plugin\API 'date' => Period\Factory::build($period->getLabel(), $date)->getLocalizedShortString() ); - $partsArray = explode(',', $parts); - if ($parts == 'all' || in_array('internalReferrers', $partsArray)) { - $this->addInternalReferrers($logAggregator, $report, $idaction, $actionType, $limitBeforeGrouping); - } - if ($parts == 'all' || in_array('followingActions', $partsArray)) { - $includeLoops = $parts != 'all' && !in_array('internalReferrers', $partsArray); - $this->addFollowingActions($logAggregator, $report, $idaction, $actionType, $limitBeforeGrouping, $includeLoops); - } - if ($parts == 'all' || in_array('externalReferrers', $partsArray)) { - $this->addExternalReferrers($logAggregator, $report, $idaction, $actionType, $limitBeforeGrouping); - } + try { + $partsArray = explode(',', $parts); + if ($parts == 'all' || in_array('internalReferrers', $partsArray)) { + $this->addInternalReferrers($logAggregator, $report, $idaction, $actionType, $limitBeforeGrouping); + } + if ($parts == 'all' || in_array('followingActions', $partsArray)) { + $includeLoops = $parts != 'all' && !in_array('internalReferrers', $partsArray); + $this->addFollowingActions($logAggregator, $report, $idaction, $actionType, $limitBeforeGrouping, $includeLoops); + } + if ($parts == 'all' || in_array('externalReferrers', $partsArray)) { + $this->addExternalReferrers($logAggregator, $report, $idaction, $actionType, $limitBeforeGrouping); + } - // derive the number of exits from the other metrics - if ($parts == 'all') { - $report['pageMetrics']['exits'] = $report['pageMetrics']['pageviews'] - - $this->getTotalTransitionsToFollowingActions() - - $report['pageMetrics']['loops']; + // derive the number of exits from the other metrics + if ($parts == 'all') { + $report['pageMetrics']['exits'] = $report['pageMetrics']['pageviews'] + - $this->getTotalTransitionsToFollowingActions() + - $report['pageMetrics']['loops']; + } + } catch (\Exception $e) { + Model::handleMaxExecutionTimeError( + Db::getReader(), + $e, + $segment->getString(), + $period->getDateStart(), + $period->getDateEnd(), + 0, + Config::getInstance()->General['live_query_max_execution_time'], + ['method' => 'Transitions.getTransitionsForAction', 'actionName' => $actionName, 'actionType' => $actionType] + ); + throw $e; } + // replace column names in the data tables $reportNames = array( 'previousPages' => true, @@ -290,7 +308,8 @@ class API extends \Piwik\Plugin\API $metrics, $rankingQuery, $joinLogActionColumn, - $secondaryOrderBy = "`name`" + $secondaryOrderBy = "`name`", + Config::getInstance()->General['live_query_max_execution_time'] ); $dataTables = $this->makeDataTablesFollowingActions($types, $data); @@ -342,7 +361,7 @@ class API extends \Piwik\Plugin\API $where = 'visit_entry_idaction_' . $type . ' = ' . intval($idaction); $metrics = array(Metrics::INDEX_NB_VISITS); - $data = $logAggregator->queryVisitsByDimension($dimensions, $where, $selects, $metrics, $rankingQuery); + $data = $logAggregator->queryVisitsByDimension($dimensions, $where, $selects, $metrics, $rankingQuery, false, Config::getInstance()->General['live_query_max_execution_time']); $referrerData = array(); $referrerSubData = array(); @@ -433,7 +452,8 @@ class API extends \Piwik\Plugin\API $metrics, $rankingQuery, $joinLogActionOn, - $secondaryOrderBy = "`name`" + $secondaryOrderBy = "`name`", + Config::getInstance()->General['live_query_max_execution_time'] ); $loops = 0; diff --git a/plugins/UserId/tests/Fixtures/TrackFewVisitsAndCreateUsers.php b/plugins/UserId/tests/Fixtures/TrackFewVisitsAndCreateUsers.php index cb390f27cf..9d463d5adb 100644 --- a/plugins/UserId/tests/Fixtures/TrackFewVisitsAndCreateUsers.php +++ b/plugins/UserId/tests/Fixtures/TrackFewVisitsAndCreateUsers.php @@ -37,6 +37,7 @@ class TrackFewVisitsAndCreateUsers extends Fixture foreach (array('user1', 'user2', 'user3') as $key => $userId) { for ($numVisits = 0; $numVisits < ($key+1) * 10; $numVisits++) { $t->setUserId($userId); + $t->setVisitorId(str_pad($numVisits.$key, 16, 'a')); if ($numVisits % 5 == 0) { $t->doTrackSiteSearch('some search term'); } diff --git a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php index e99d495931..b75d09ad77 100644 --- a/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php +++ b/tests/PHPUnit/Integration/ArchiveProcessor/LoaderTest.php @@ -10,10 +10,12 @@ namespace Piwik\Tests\Integration\ArchiveProcessor; +use Piwik\Archive\ArchiveInvalidator; use Piwik\ArchiveProcessor\Parameters; use Piwik\ArchiveProcessor\Loader; use Piwik\Common; use Piwik\Config; +use Piwik\Container\StaticContainer; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\DataAccess\ArchiveWriter; use Piwik\Date; @@ -31,6 +33,7 @@ class LoaderTest extends IntegrationTestCase parent::beforeTableDataCached(); Fixture::createWebsite('2012-02-03 00:00:00'); + Fixture::createWebsite('2012-02-03 00:00:00'); } public function test_loadExistingArchiveIdFromDb_returnsFalsesIfNoArchiveFound() @@ -40,7 +43,7 @@ class LoaderTest extends IntegrationTestCase $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals([false, false, false], $archiveInfo); + $this->assertEquals([false, false, false, false], $archiveInfo); } /** @@ -55,12 +58,12 @@ class LoaderTest extends IntegrationTestCase $loader = new Loader($params); $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertNotEquals([false, false, false], $archiveInfo); + $this->assertNotEquals([false, false, false, false], $archiveInfo); Config::getInstance()->Debug[$configSetting] = 1; $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals([false, false, false], $archiveInfo); + $this->assertEquals([false, false, false, false], $archiveInfo); } public function getTestDataForLoadExistingArchiveIdFromDbDebugConfig() @@ -82,7 +85,7 @@ class LoaderTest extends IntegrationTestCase $loader = new Loader($params); $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals(['1', '10', '0'], $archiveInfo); + $this->assertEquals(['1', '10', '0', true], $archiveInfo); } public function test_loadExistingArchiveIdFromDb_returnsArchiveIfForACurrentPeriod_AndNewEnough() @@ -93,7 +96,7 @@ class LoaderTest extends IntegrationTestCase $loader = new Loader($params); $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals(['1', '10', '0'], $archiveInfo); + $this->assertEquals(['1', '10', '0', true], $archiveInfo); } public function test_loadExistingArchiveIdFromDb_returnsNoArchiveIfForACurrentPeriod_AndNoneAreNewEnough() @@ -104,7 +107,106 @@ class LoaderTest extends IntegrationTestCase $loader = new Loader($params); $archiveInfo = $loader->loadExistingArchiveIdFromDb(); - $this->assertEquals([false, false, false], $archiveInfo); + $this->assertEquals([false, '10', '0', true], $archiveInfo); // visits are still returned as this was the original behavior + } + + /** + * @dataProvider getTestDataForGetReportsToInvalidate + */ + public function test_getReportsToInvalidate_returnsCorrectReportsToInvalidate($rememberedReports, $idSite, $period, $date, $segment, $expected) + { + $invalidator = StaticContainer::get(ArchiveInvalidator::class); + foreach ($rememberedReports as $entry) { + $invalidator->rememberToInvalidateArchivedReportsLater($entry['idSite'], Date::factory($entry['date'])); + } + + $params = new Parameters(new Site($idSite), Factory::build($period, $date), new Segment($segment, [$idSite])); + $loader = new Loader($params); + + $reportsToInvalidate = $loader->getReportsToInvalidate(); + foreach ($reportsToInvalidate as &$sites) { + sort($sites); + } + $this->assertEquals($expected, $reportsToInvalidate); + } + + public function getTestDataForGetReportsToInvalidate() + { + return [ + // two dates for one site + [ + [ + ['idSite' => 1, 'date' => '2013-04-05'], + ['idSite' => 1, 'date' => '2013-03-05'], + ['idSite' => 2, 'date' => '2013-05-05'], + ], + 1, + 'day', + '2013-04-05', + '', + [ + '2013-04-05' => [1], + ], + ], + + // no dates for a site + [ + [ + ['idSite' => '', 'date' => '2013-04-05'], + ['idSite' => '', 'date' => '2013-04-06'], + ['idSite' => 2, 'date' => '2013-05-05'], + ], + 1, + 'day', + '2013-04-05', + 'browserCode==ff', + [], + ], + + // day period not within range + [ + [ + ['idSite' => 1, 'date' => '2014-03-04'], + ['idSite' => 1, 'date' => '2014-03-06'], + ], + 1, + 'day', + '2013-03-05', + '', + [], + ], + + // non-day periods + [ + [ + ['idSite' => 1, 'date' => '2014-03-01'], + ['idSite' => 1, 'date' => '2014-03-06'], + ['idSite' => 2, 'date' => '2014-03-01'], + ], + 1, + 'week', + '2014-03-01', + '', + [ + '2014-03-01' => [1, 2], + ], + ], + [ + [ + ['idSite' => 1, 'date' => '2014-02-01'], + ['idSite' => 1, 'date' => '2014-03-06'], + ['idSite' => 2, 'date' => '2014-03-05'], + ['idSite' => 2, 'date' => '2014-03-06'], + ], + 1, + 'month', + '2014-03-01', + '', + [ + '2014-03-06' => [1, 2], + ], + ], + ]; } private function insertArchive(Parameters $params, $tsArchived = null, $visits = 10) diff --git a/tests/PHPUnit/Integration/ArchiveTest.php b/tests/PHPUnit/Integration/ArchiveTest.php deleted file mode 100644 index 9097e82664..0000000000 --- a/tests/PHPUnit/Integration/ArchiveTest.php +++ /dev/null @@ -1,441 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link https://matomo.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - */ -namespace Piwik\Tests\Integration; - -use Piwik\API\Proxy; -use Piwik\API\Request; -use Piwik\Archive as PiwikArchive; -use Piwik\ArchiveProcessor; -use Piwik\ArchiveProcessor\Parameters; -use Piwik\ArchiveProcessor\Rules; -use Piwik\Common; -use Piwik\Config; -use Piwik\DataAccess\ArchiveSelector; -use Piwik\DataAccess\ArchiveTableCreator; -use Piwik\DataAccess\ArchiveWriter; -use Piwik\DataAccess\LogAggregator; -use Piwik\Date; -use Piwik\Db; -use Piwik\Piwik; -use Piwik\Plugins\UserLanguage; -use Piwik\Segment; -use Piwik\Site; -use Piwik\Tests\Fixtures\OneVisitorTwoVisits; -use Piwik\Tests\Framework\Fixture; -use Piwik\Tests\Framework\TestCase\IntegrationTestCase; -use Piwik\Period\Factory as PeriodFactory; -use Piwik\Archive\Chunk; - -class Archive extends PiwikArchive -{ - public function get($archiveNames, $archiveDataType, $idSubtable = null) - { - return parent::get($archiveNames, $archiveDataType, $idSubtable); - } -} - -class CustomArchiveQueryFactory extends PiwikArchive\ArchiveQueryFactory -{ - public function newInstance(\Piwik\Archive\Parameters $params, $forceIndexedBySite, $forceIndexedByDate) - { - return new Archive($params, $forceIndexedBySite, $forceIndexedByDate); - } -} - -/** - * @group Core - */ -class ArchiveTest extends IntegrationTestCase -{ - /** - * @var OneVisitorTwoVisits - */ - public static $fixture; - - public function tearDown(): void - { - parent::tearDown(); - - unset($_GET['trigger']); - } - - protected static function configureFixture($fixture) - { - $fixture->createSuperUser = true; - } - - protected static function beforeTableDataCached() - { - $date = Date::factory('2010-03-01'); - - $archiveTableCreator = new ArchiveTableCreator(); - $archiveTableCreator->getBlobTable($date); - $archiveTableCreator->getNumericTable($date); - } - - public function getForceOptionsForForceArchivingOnBrowserRequest() - { - return array( - array(1), - array(null) - ); - } - - /** - * @dataProvider getForceOptionsForForceArchivingOnBrowserRequest - */ - public function test_ArchivingIsLaunchedForRanges_WhenForceOnBrowserRequest_IsTruthy($optionValue) - { - $this->archiveDataForIndividualDays(); - - Config::getInstance()->General['archiving_range_force_on_browser_request'] = $optionValue; - Rules::setBrowserTriggerArchiving(false); - - $data = $this->initiateArchivingForRange(); - - $this->assertNotEmpty($data); - $this->assertArchiveTablesAreNotEmpty('2010_03'); - } - - public function test_ArchivingIsNotLaunchedForRanges_WhenForceOnBrowserRequest_IsFalse() - { - $this->archiveDataForIndividualDays(); - - Config::getInstance()->General['archiving_range_force_on_browser_request'] = 0; - Rules::setBrowserTriggerArchiving(false); - - $data = $this->initiateArchivingForRange(); - - $this->assertEmpty($data); - $this->assertArchiveTablesAreEmpty('2010_03'); - } - - public function test_ArchiveIsLaunched_WhenForceOnBrowserRequest_IsFalse_AndArchivePhpTriggered() - { - $this->archiveDataForIndividualDays(); - - Config::getInstance()->General['archiving_range_force_on_browser_request'] = 0; - $_GET['trigger'] = 'archivephp'; - Rules::setBrowserTriggerArchiving(false); - - $data = $this->initiateArchivingForRange(); - - $this->assertNotEmpty($data); - $this->assertArchiveTablesAreNotEmpty('2010_03'); - } - - public function test_ArchiveBlob_ShouldBeAbleToLoadFirstLevelDataArrays() - { - $this->createManyDifferentArchiveBlobs(); - - $archive = $this->getArchive('day', '2013-01-01,2013-01-05'); - $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob'); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl' => 'test01')); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl' => 'test02')); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl' => 'test03')); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl' => 'test04')); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl' => 0)); - } - - public function test_ArchiveBlob_ShouldBeAbleToLoadOneSubtable_NoMatterWhetherTheyAreStoredSeparatelyOrInACombinedSubtableEntry() - { - $this->createManyDifferentArchiveBlobs(); - - $archive = $this->getArchive('day', '2013-01-01,2013-01-05'); - $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', 2); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_2' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl_2' => 'test2')); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl_2' => 'subtable2')); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_2' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_2' => 0)); - - // test another one - $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', 5); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_5' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl_5' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl_5' => 'subtable5')); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_5' => 'subtable45')); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_5' => 0)); - - // test one that does not exist - $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', 999); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_999' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl_999' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl_999' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_999' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_999' => 0)); - } - - public function test_ArchiveBlob_ShouldBeAbleToLoadAllSubtables_NoMatterWhetherTheyAreStoredSeparatelyOrInACombinedSubtableEntry() - { - $this->createManyDifferentArchiveBlobs(); - - $archive = $this->getArchive('day', '2013-01-01,2013-01-06'); - $dataArrays = $archive->get(array('Actions_Actionsurl'), 'blob', Archive::ID_SUBTABLE_LOAD_ALL_SUBTABLES); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl' => 'test01')); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actionsurl' => 'test02', 'Actions_Actionsurl_1' => 'test1', 'Actions_Actionsurl_2' => 'test2')); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actionsurl' => 'test03', 'Actions_Actionsurl_1' => 'subtable1', 'Actions_Actionsurl_2' => 'subtable2', 'Actions_Actionsurl_5' => 'subtable5')); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl' => 'test04', 'Actions_Actionsurl_5' => 'subtable45', 'Actions_Actionsurl_6' => 'subtable6')); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-06', array('Actions_Actionsurl' => 'test06')); - } - - public function test_ArchiveBlob_ShouldBeAbleToLoadDifferentArchives_NoMatterWhetherTheyAreStoredSeparatelyOrInACombinedSubtableEntry() - { - $this->createManyDifferentArchiveBlobs(); - - $archive = $this->getArchive('day', '2013-01-01,2013-01-06'); - $dataArrays = $archive->get(array('Actions_Actionsurl', 'Actions_Actions'), 'blob', 2); - - $this->assertArchiveBlob($dataArrays, '2013-01-01', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-02', array('Actions_Actions_2' => 'actionsSubtable2', 'Actions_Actionsurl_2' => 'test2')); - $this->assertArchiveBlob($dataArrays, '2013-01-03', array('Actions_Actions_2' => 'actionsTest2', 'Actions_Actionsurl_2' => 'subtable2')); - $this->assertArchiveBlob($dataArrays, '2013-01-04', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-05', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0)); - $this->assertArchiveBlob($dataArrays, '2013-01-06', array('Actions_Actionsurl_2' => 0, 'Actions_Actions_2' => 0)); - } - - /** - * @dataProvider findBlobsWithinDifferentChunksDataProvider - */ - public function test_ArchiveBlob_ShouldBeFindBlobs_WithinDifferentChunks($idSubtable, $expectedBlob) - { - $recordName = 'Actions_Actions'; - - $chunk = new Chunk(); - $chunk5 = $chunk->getRecordNameForTableId($recordName, $subtableId = 5); - $chunk152 = $chunk->getRecordNameForTableId($recordName, $subtableId = 152); - $chunk399 = $chunk->getRecordNameForTableId($recordName, $subtableId = 399); - - $this->createArchiveBlobEntry('2013-01-02', array( - $recordName => 'actions_02', - $chunk5 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')), - $chunk152 => serialize(array(151 => 'actionsSubtable151', 152 => 'actionsSubtable152')), - $chunk399 => serialize(array(399 => 'actionsSubtable399')) - )); - - $archive = $this->getArchive('day', '2013-01-02,2013-01-02'); - - $dataArrays = $archive->get(array('Actions_Actions'), 'blob', $idSubtable); - $this->assertArchiveBlob($dataArrays, '2013-01-02', $expectedBlob); - } - - public function findBlobsWithinDifferentChunksDataProvider() - { - return array( - array($idSubtable = 2, $expectedBlobs = array('Actions_Actions_2' => 'actionsSubtable2')), - array(5, array('Actions_Actions_5' => 'actionsSubtable5')), - array(151, array('Actions_Actions_151' => 'actionsSubtable151')), - array(152, array('Actions_Actions_152' => 'actionsSubtable152')), - array(399, array('Actions_Actions_399' => 'actionsSubtable399')), - // this one does not exist - array(404, array('Actions_Actions_404' => 0)), - ); - } - - public function testExistingArchivesAreReplaced() - { - $date = self::$fixture->dateTime; - $period = PeriodFactory::makePeriodFromQueryParams('UTC', 'day', $date); - - // request an report to trigger archiving - $userLanguageReport = Proxy::getInstance()->call('\\Piwik\\Plugins\\UserLanguage\\API', 'getLanguage', array( - 'idSite' => 1, - 'period' => 'day', - 'date' => $date - )); - - $this->assertEquals(1, $userLanguageReport->getRowsCount()); - $this->assertEquals('UserLanguage_LanguageCode fr', $userLanguageReport->getFirstRow()->getColumn('label')); - $this->assertEquals('UserLanguage_LanguageCode fr', $userLanguageReport->getLastRow()->getColumn('label')); - - $parameters = new Parameters(new Site(1), $period, new Segment('', [])); - $parameters->setRequestedPlugin('UserLanguage'); - - $result = ArchiveSelector::getArchiveIdAndVisits($parameters, $period->getDateStart()->getDateStartUTC()); - $idArchive = $result ? array_shift($result) : null; - - if (empty($idArchive)) { - $this->fail('Archive should be available'); - } - - // track a new visits now - $t = Fixture::getTracker(1, $date, $defaultInit = true); - $t->setForceVisitDateTime(Date::factory($date)->addHour(1)->getDatetime()); - $t->setUrl('http://site.com/index.htm'); - $t->setBrowserLanguage('pt-br'); - Fixture::checkResponse($t->doTrackPageView('my_site')); - - $archiveWriter = new ArchiveWriter($parameters, !!$idArchive); - $archiveWriter->idArchive = $idArchive; - - $archiveProcessor = new ArchiveProcessor($parameters, $archiveWriter, - new LogAggregator($parameters)); - - $archiveProcessor->setNumberOfVisits(1, 1); - - // directly trigger specific archiver for existing archive - $archiver = new UserLanguage\Archiver($archiveProcessor); - $archiver->aggregateDayReport(); - $archiveWriter->finalizeArchive(); - - // report should be updated - $userLanguageReport = Proxy::getInstance()->call('\\Piwik\\Plugins\\UserLanguage\\API', 'getLanguage', array( - 'idSite' => 1, - 'period' => 'day', - 'date' => $date - )); - - $this->assertEquals(2, $userLanguageReport->getRowsCount()); - $this->assertEquals('UserLanguage_LanguageCode fr', $userLanguageReport->getFirstRow()->getColumn('label')); - $this->assertEquals('UserLanguage_LanguageCode pt', $userLanguageReport->getLastRow()->getColumn('label')); - } - - public function testNoFutureArchiveTablesAreCreatedWithoutArchiving() - { - $dateTime = '2066-01-01'; - Config::getInstance()->General['enable_browser_archiving_triggering'] = 0; - - // request API which should not trigger archiving due to config and shouldn't create any archive tables - Request::processRequest('VisitsSummary.get', array('idSite' => 1, 'period' => 'day', 'date' => $dateTime)); - - $numericTables = Db::get()->fetchAll('SHOW TABLES like "%archive_numeric_2066_%"'); - - $this->assertEmpty($numericTables, 'Archive table for future date found'); - } - - public function testNoFutureArchiveTablesAreCreatedWithArchiving() - { - $dateTime = '2066-01-01'; - Config::getInstance()->General['enable_browser_archiving_triggering'] = 1; - - // request API which should not trigger archiving due to config and shouldn't create any archive tables - Request::processRequest('VisitsSummary.get', array('idSite' => 1, 'period' => 'day', 'date' => $dateTime)); - - $numericTables = Db::get()->fetchAll('SHOW TABLES like "%archive_numeric_2066_%"'); - - $this->assertEmpty($numericTables, 'Archive table for future date found'); - } - - private function createManyDifferentArchiveBlobs() - { - $recordName1 = 'Actions_Actions'; - $recordName2 = 'Actions_Actionsurl'; - - $chunk = new Chunk(); - $chunk0_1 = $chunk->getRecordNameForTableId($recordName1, 0); - $chunk0_2 = $chunk->getRecordNameForTableId($recordName2, 0); - - $this->createArchiveBlobEntry('2013-01-01', array( - $recordName2 => 'test01' - )); - $this->createArchiveBlobEntry('2013-01-02', array( - $recordName2 => 'test02', - $recordName2 . '_1' => 'test1', // testing BC where each subtable was stored seperately - $recordName2 . '_2' => 'test2', // testing BC - $recordName1 => 'actions_02', - $chunk0_1 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')) - )); - $this->createArchiveBlobEntry('2013-01-03', array( - $recordName2 => 'test03', - $chunk0_2 => serialize(array(1 => 'subtable1', 2 => 'subtable2', 5 => 'subtable5')), - $recordName1 => 'actions_03', - $recordName1 . '_1' => 'actionsTest1', - $recordName1 . '_2' => 'actionsTest2' - )); - $this->createArchiveBlobEntry('2013-01-04', array( - $recordName2 => 'test04', - $recordName2 . '_5' => 'subtable45', - $recordName2 . '_6' => 'subtable6' - )); - $this->createArchiveBlobEntry('2013-01-06', array( - $recordName2 => 'test06', - $chunk0_2 => serialize(array()) - )); - } - - private function assertArchiveBlob(PiwikArchive\DataCollection $dataCollection, $date, $expectedBlob) - { - $dateIndex = $date . ',' . $date; - $dataArrays = $dataCollection->get(1, $dateIndex); - - if (!empty($expectedBlob) && 0 !== reset($expectedBlob)) { - $this->assertNotEmpty($dataArrays['_metadata']['ts_archived']); - $dataArrays['_metadata']['ts_archived'] = true; - unset($dataArrays['_metadata']); - } - - $this->assertEquals($expectedBlob, $dataArrays); - } - - private function createArchiveBlobEntry($date, $blobs) - { - $oPeriod = PeriodFactory::makePeriodFromQueryParams('UTC', 'day', $date); - - $segment = new Segment(false, array(1)); - $params = new Parameters(new Site(1), $oPeriod, $segment); - $writer = new ArchiveWriter($params, false); - $writer->initNewArchive(); - foreach ($blobs as $name => $blob) { - $writer->insertBlobRecord($name, $blob); - } - $writer->finalizeArchive(); - } - - private function assertArchiveTablesAreNotEmpty($tableMonth) - { - $this->assertNotEquals(0, $this->getRangeArchiveTableCount('archive_numeric', $tableMonth)); - } - - private function assertArchiveTablesAreEmpty($tableMonth) - { - $this->assertEquals(0, $this->getRangeArchiveTableCount('archive_numeric', $tableMonth)); - $this->assertEquals(0, $this->getRangeArchiveTableCount('archive_blob', $tableMonth)); - } - - private function getRangeArchiveTableCount($tableType, $tableMonth) - { - $table = Common::prefixTable($tableType . '_' . $tableMonth); - return Db::fetchOne("SELECT COUNT(*) FROM $table WHERE period = " . Piwik::$idPeriods['range']); - } - - private function initiateArchivingForRange() - { - $archive = $this->getArchive('range'); - return $archive->getNumeric('nb_visits'); - } - - private function archiveDataForIndividualDays() - { - $archive = $this->getArchive('day'); - return $archive->getNumeric('nb_visits'); - } - - /** - * @return Archive - */ - private function getArchive($period, $day = '2010-03-04,2010-03-07') - { - /** @noinspection PhpIncompatibleReturnTypeInspection */ - return Archive::build(self::$fixture->idSite, $period, $day); - } - - public function provideContainerConfig() - { - return [ - PiwikArchive\ArchiveQueryFactory::class => \DI\object(CustomArchiveQueryFactory::class), - ]; - } -} - -ArchiveTest::$fixture = new OneVisitorTwoVisits();
\ No newline at end of file diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php index 775c348e68..bb83400022 100644 --- a/tests/PHPUnit/Integration/CronArchiveTest.php +++ b/tests/PHPUnit/Integration/CronArchiveTest.php @@ -10,7 +10,9 @@ namespace Piwik\Tests\Integration; use Piwik\Container\StaticContainer; use Piwik\CronArchive; +use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; +use Piwik\Db; use Piwik\Plugins\CoreAdminHome\tests\Framework\Mock\API; use Piwik\Plugins\SegmentEditor\Model; use Piwik\Tests\Framework\Fixture; @@ -40,12 +42,27 @@ class CronArchiveTest extends IntegrationTestCase $cronarchive->setApiToInvalidateArchivedReport($api); $cronarchive->init(); - $expectedInvalidations = array( - array(array(1,2), '2014-04-05'), - array(array(2), '2014-04-06') - ); + /** + * should look like this but the result is random + * array( + array(array(1,2), '2014-04-05'), + array(array(2), '2014-04-06') + ) + */ + $invalidatedReports = $api->getInvalidatedReports(); + $this->assertCount(2, $invalidatedReports); + sort($invalidatedReports[0][0]); + sort($invalidatedReports[1][0]); + usort($invalidatedReports, function ($a, $b) { + return strcmp($a[1], $b[1]); + }); + + $this->assertSame(array(1,2), $invalidatedReports[0][0]); + $this->assertSame('2014-04-05', $invalidatedReports[0][1]); + + $this->assertSame(array(2), $invalidatedReports[1][0]); + $this->assertSame('2014-04-06', $invalidatedReports[1][1]); - $this->assertEquals($expectedInvalidations, $api->getInvalidatedReports()); } public function test_setSegmentsToForceFromSegmentIds_CorrectlyGetsSegmentDefinitions_FromSegmentIds() @@ -89,7 +106,7 @@ class CronArchiveTest extends IntegrationTestCase \Piwik\Tests\Framework\Mock\FakeCliMulti::$specifiedResults = array( '/method=API.get/' => serialize(array(array('nb_visits' => 1))) ); - + Fixture::createWebsite('2014-12-12 00:01:02'); SegmentAPI::getInstance()->add('foo', 'actions>=1', 1, true, true); $id = SegmentAPI::getInstance()->add('barb', 'actions>=2', 1, true, true); @@ -215,12 +232,198 @@ LOG; self::assertStringContainsString($expected, $logger->output); } + /** + * @dataProvider getTestDataForIsThereAValidArchiveForPeriod + */ + public function test_isThereAValidArchiveForPeriod_modifiesLastNCorrectly($archiveRows, $idSite, $period, $date, $expected) + { + Date::$now = strtotime('2015-03-04 05:05:05'); + + Fixture::createWebsite('2014-12-12 00:01:02'); + + $this->insertArchiveData($archiveRows); + + $cronArchive = new CronArchive(); + $result = $cronArchive->isThereAValidArchiveForPeriod($idSite, $period, $date); + + $this->assertEquals($expected, $result); + } + + public function getTestDataForIsThereAValidArchiveForPeriod() + { + return [ + // single periods that include today (we don't perform the check and just archive since we assume today is invalid) + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1], + ], + 1, + 'day', + '2015-03-04', + [false, null], + ], + [ + [], + 1, + 'week', + '2015-03-04', + [false, null], + ], + [ + [], + 1, + 'month', + '2015-03-04', + [false, null], + ], + [ + [], + 1, + 'year', + '2015-03-04', + [false, null], + ], + + // single periods that do not include today + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-02', 'name' => 'done', 'value' => 1], + ], + 1, + 'day', + '2015-03-02', + [true, '2015-03-02'], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-01', 'name' => 'done', 'value' => 1], + ], + 1, + 'day', + '2015-03-02', + [false, '2015-03-02'], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 2, 'date1' => '2015-02-16', 'date2' => '2015-02-22', 'name' => 'done', 'value' => 1], + ], + 1, + 'week', + '2015-02-17', + [true, '2015-02-17'], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 2, 'date1' => '2015-02-15', 'date2' => '2015-02-21', 'name' => 'done', 'value' => 4], + ], + 1, + 'week', + '2015-02-17', + [false, '2015-02-17'], + ], + + // lastN periods + [ // last day invalid, some valid in between + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-03', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1], + ['idarchive' => 5, 'idsite' => 1, 'period' => 1, 'date1' => '2015-02-28', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4], + ], + 1, + 'day', + 'last5', + [false, 'last5'], + ], + [ // last two invalid, rest valid + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 4], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-03', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 4], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-02', 'name' => 'done', 'value' => 1], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-01', 'name' => 'done', 'value' => 1], + ['idarchive' => 5, 'idsite' => 1, 'period' => 1, 'date1' => '2015-02-28', 'date2' => '2015-02-28', 'name' => 'done', 'value' => 1], + ], + 1, + 'day', + 'last5', + [false, 'last2'], + ], + [ // all valid + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-04', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-03', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 1], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-02', 'date2' => '2015-03-02', 'name' => 'done', 'value' => 1], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2015-03-01', 'date2' => '2015-03-01', 'name' => 'done', 'value' => 1], + ['idarchive' => 5, 'idsite' => 1, 'period' => 1, 'date1' => '2015-02-28', 'date2' => '2015-02-28', 'name' => 'done', 'value' => 1], + ], + 1, + 'day', + 'last5', + [false, 'last2'], + ], + [ // month w/ last 3 invalid, today valid + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 3, 'date1' => '2015-03-01', 'date2' => '2015-03-31', 'name' => 'done', 'value' => 1], + ['idarchive' => 2, 'idsite' => 1, 'period' => 3, 'date1' => '2015-02-01', 'date2' => '2015-02-28', 'name' => 'done', 'value' => 4], + ['idarchive' => 3, 'idsite' => 1, 'period' => 3, 'date1' => '2015-01-01', 'date2' => '2015-01-31', 'name' => 'done', 'value' => 4], + ['idarchive' => 4, 'idsite' => 1, 'period' => 3, 'date1' => '2014-12-01', 'date2' => '2014-12-31', 'name' => 'done', 'value' => 1], + ['idarchive' => 5, 'idsite' => 1, 'period' => 3, 'date1' => '2014-11-01', 'date2' => '2014-11-30', 'name' => 'done', 'value' => 1], + ], + 1, + 'month', + 'last5', + [false, 'last3'], + ], + + // range periods + [ // includes today + [ + ['idarchive' => 5, 'idsite' => 1, 'period' => 5, 'date1' => '2015-03-02', 'date2' => '2015-03-04', 'name' => 'done', 'value' => 1], + ], + 1, + 'range', + '2015-03-02,2015-03-04', + [false, null], + ], + [ // does not include today, invalid + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2015-03-01', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 4], + ], + 1, + 'range', + '2015-03-01,2015-03-03', + [false, '2015-03-01,2015-03-03'], + ], + [ // does not include today, valid + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 5, 'date1' => '2015-03-01', 'date2' => '2015-03-03', 'name' => 'done', 'value' => 1], + ], + 1, + 'range', + '2015-03-01,2015-03-03', + [true, '2015-03-01,2015-03-03'], + ], + ]; + } + public function provideContainerConfig() { return array( 'Piwik\CliMulti' => \DI\object('Piwik\Tests\Framework\Mock\FakeCliMulti') ); } + + private function insertArchiveData($archiveRows) + { + foreach ($archiveRows as $row) { + $table = ArchiveTableCreator::getNumericTable(Date::factory($row['date1'])); + + $tsArchived = isset($row['ts_archived']) ? $row['ts_archived'] : Date::now()->getDatetime(); + Db::query("INSERT INTO `$table` (idarchive, idsite, period, date1, date2, `name`, `value`, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + [$row['idarchive'], $row['idsite'], $row['period'], $row['date1'], $row['date2'], $row['name'], $row['value'], $tsArchived]); + } + } } class TestCronArchive extends CronArchive diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php index e5423c371d..13d70cb1c2 100644 --- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php @@ -83,11 +83,12 @@ class ArchiveInvalidatorTest extends IntegrationTestCase { //Updated for change to allow for multiple transactions to invalidate the same report without deadlock. $key = 'report_to_invalidate_2_2014-04-05' . '_' . getmypid(); - $this->assertFalse(Option::get($key)); + $this->assertEmpty(Option::getLike('%'. $key . '%')); - $this->rememberReport(2, '2014-04-05'); + $keyStored = $this->rememberReport(2, '2014-04-05'); - $this->assertSame('1', Option::get($key)); + $this->assertStringEndsWith($key, $keyStored); + $this->assertSame('1', Option::get($keyStored)); } public function test_rememberToInvalidateArchivedReportsLater_shouldNotCreateEntryTwice() @@ -96,7 +97,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $this->rememberReport(2, '2014-04-05'); $this->rememberReport(2, '2014-04-05'); - $this->assertCount(1, Option::getLike('report_to_invalidate%')); + $this->assertCount(1, Option::getLike('%report_to_invalidate%')); } public function test_getRememberedArchivedReportsThatShouldBeInvalidated_shouldNotReturnEntriesInCaseNoneAreRemembered() @@ -112,7 +113,22 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); - $this->assertSame($this->getRememberedReportsByDate(), $reports); + $this->assertSameReports($this->getRememberedReportsByDate(), $reports); + } + + private function assertSameReports($expected, $actual) + { + $keys1 = array_keys($expected); + $keys2 = array_keys($actual); + sort($keys1); + sort($keys2); + + $this->assertSame($keys1, $keys2); + foreach ($expected as $index => $values) { + sort($values); + sort($actual[$index]); + $this->assertSame($values, $actual[$index]); + } } public function test_forgetRememberedArchivedReportsToInvalidateForSite_shouldNotDeleteAnythingInCaseNoReportForThatSite() @@ -122,7 +138,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase $this->invalidator->forgetRememberedArchivedReportsToInvalidateForSite(10); $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); - $this->assertSame($this->getRememberedReportsByDate(), $reports); + $this->assertSameReports($this->getRememberedReportsByDate(), $reports); } public function test_forgetRememberedArchivedReportsToInvalidateForSite_shouldOnlyDeleteReportsBelongingToThatSite() @@ -137,7 +153,7 @@ class ArchiveInvalidatorTest extends IntegrationTestCase '2014-05-05' => array(2, 5), '2014-04-06' => array(3) ); - $this->assertSame($expected, $reports); + $this->assertSameReports($expected, $reports); } public function test_forgetRememberedArchivedReportsToInvalidate_shouldNotForgetAnythingIfThereIsNoMatch() @@ -147,12 +163,12 @@ class ArchiveInvalidatorTest extends IntegrationTestCase // site does not match $this->invalidator->forgetRememberedArchivedReportsToInvalidate(10, Date::factory('2014-04-05')); $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); - $this->assertSame($this->getRememberedReportsByDate(), $reports); + $this->assertSameReports($this->getRememberedReportsByDate(), $reports); // date does not match $this->invalidator->forgetRememberedArchivedReportsToInvalidate(7, Date::factory('2012-04-05')); $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); - $this->assertSame($this->getRememberedReportsByDate(), $reports); + $this->assertSameReports($this->getRememberedReportsByDate(), $reports); } public function test_forgetRememberedArchivedReportsToInvalidate_shouldOnlyDeleteReportBelongingToThatSiteAndDate() @@ -169,13 +185,13 @@ class ArchiveInvalidatorTest extends IntegrationTestCase '2014-04-08' => array(7), '2014-05-08' => array(7), ); - $this->assertSame($expected, $reports); + $this->assertSameReports($expected, $reports); unset($expected['2014-05-08']); $this->invalidator->forgetRememberedArchivedReportsToInvalidate(7, Date::factory('2014-05-08')); $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); - $this->assertSame($expected, $reports); + $this->assertSameReports($expected, $reports); } public function test_markArchivesAsInvalidated_shouldForgetInvalidatedSitesAndDates() @@ -198,21 +214,21 @@ class ArchiveInvalidatorTest extends IntegrationTestCase '2014-04-06' => array(3), '2014-05-08' => array(7), ); - $this->assertSame($expected, $reports); + $this->assertSameReports($expected, $reports); } private function rememberReport($idSite, $date) { $date = Date::factory($date); - $this->invalidator->rememberToInvalidateArchivedReportsLater($idSite, $date); + return $this->invalidator->rememberToInvalidateArchivedReportsLater($idSite, $date); } private function getRememberedReportsByDate() { return array( - '2014-04-05' => array(1, 2, 4, 7), - '2014-05-05' => array(2, 5), '2014-04-06' => array(3), + '2014-04-05' => array(4, 7, 2, 1), + '2014-05-05' => array(5, 2), '2014-04-08' => array(7), '2014-05-08' => array(7), ); diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php new file mode 100644 index 0000000000..6143534a87 --- /dev/null +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveSelectorTest.php @@ -0,0 +1,212 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link https://matomo.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ + +namespace Piwik\Tests\Integration\DataAccess; + + +use Piwik\ArchiveProcessor\Parameters; +use Piwik\DataAccess\ArchiveSelector; +use Piwik\DataAccess\ArchiveTableCreator; +use Piwik\Date; +use Piwik\Db; +use Piwik\Period\Factory; +use Piwik\Segment; +use Piwik\Site; +use Piwik\Tests\Framework\Fixture; +use Piwik\Tests\Framework\TestCase\IntegrationTestCase; + +class ArchiveSelectorTest extends IntegrationTestCase +{ + protected static function configureFixture($fixture) + { + parent::configureFixture($fixture); + $fixture->createSuperUser = true; + } + + /** + * @dataProvider getTestDataForGetArchiveIdAndVisits + */ + public function test_getArchiveIdAndVisits_returnsCorrectResult($archiveRows, $segment, $minDateProcessed, $includeInvalidated, $expected) + { + Fixture::createWebsite('2010-02-02 00:00:00'); + + $this->insertArchiveData($archiveRows); + + $params = new Parameters(new Site(1), Factory::build('day', '2019-10-05'), new Segment($segment, [1])); + $result = ArchiveSelector::getArchiveIdAndVisits($params, $minDateProcessed, $includeInvalidated); + $this->assertEquals($expected, $result); + } + + private function insertArchiveData($archiveRows) + { + $table = ArchiveTableCreator::getNumericTable(Date::factory('2019-10-01 12:13:14')); + foreach ($archiveRows as $row) { + $tsArchived = isset($row['ts_archived']) ? $row['ts_archived'] : Date::now()->getDatetime(); + Db::query("INSERT INTO `$table` (idarchive, idsite, period, date1, date2, `name`, `value`, ts_archived) VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + [$row['idarchive'], $row['idsite'], $row['period'], $row['date1'], $row['date2'], $row['name'], $row['value'], $tsArchived]); + } + } + + public function getTestDataForGetArchiveIdAndVisits() + { + $minDateProcessed = Date::factory('2020-03-04 00:00:00')->subSeconds(900)->getDatetime(); + return [ + // no archive data found + [ // nothing in the db + [], + '', + $minDateProcessed, + true, + [false, false, false, false], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 2, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-06', 'date2' => '2019-10-06', 'name' => 'done', 'value' => 1], + ['idarchive' => 3, 'idsite' => 2, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1], + ], + '', + $minDateProcessed, + true, + [false, false, false, false], + ], + + // value is not valid + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 4], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 2], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 3], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 99], + ], + '', + $minDateProcessed, + false, + [false, false, false, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 4], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 20], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 40], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 2], + ['idarchive' => 3, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 3], + ['idarchive' => 4, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 99], + ], + '', + $minDateProcessed, + false, + [false, 0, 0, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 4], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 20], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 40], + ], + '', + $minDateProcessed, + false, + [false, 20, 40, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 1], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 30], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 50], + ], + '', + $minDateProcessed, + false, + [false, 30, 50, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 1], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 30], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 50], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done.VisitsSummary', 'value' => 4], + ], + '', + $minDateProcessed, + false, + [false, 0, 0, true], + ], + + // archive is too old + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1, + 'ts_archived' => Date::factory($minDateProcessed)->subSeconds(1)->getDatetime()], + ], + '', + $minDateProcessed, + false, + [false, false, false, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1, + 'ts_archived' => Date::factory($minDateProcessed)->subSeconds(1)->getDatetime()], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 1, + 'ts_archived' => Date::factory($minDateProcessed)->subSeconds(1)->getDatetime()], + ], + '', + $minDateProcessed, + false, + [false, 1, false, true], + ], + + // no archive done flags, but metric + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 10], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 1], + ], + '', + $minDateProcessed, + false, + [false, false, false, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 10], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 3], + ['idarchive' => 2, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 5], + ], + '', + $minDateProcessed, + false, + [false, false, false, true], + ], + + // archive exists and is usable + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1], + ], + '', + $minDateProcessed, + false, + [1, false, false, true], + ], + [ + [ + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'done', 'value' => 1], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits', 'value' => 5], + ['idarchive' => 1, 'idsite' => 1, 'period' => 1, 'date1' => '2019-10-05', 'date2' => '2019-10-05', 'name' => 'nb_visits_converted', 'value' => 10], + ], + '', + $minDateProcessed, + false, + [1, 5, 10, true], + ], + ]; + } +}
\ No newline at end of file diff --git a/tests/PHPUnit/Integration/FrontControllerTest.php b/tests/PHPUnit/Integration/FrontControllerTest.php index a0417c3c43..ca666618d2 100644 --- a/tests/PHPUnit/Integration/FrontControllerTest.php +++ b/tests/PHPUnit/Integration/FrontControllerTest.php @@ -48,7 +48,7 @@ FORMAT; $this->assertEquals('error', $response['result']); $expectedFormat = <<<FORMAT -test message on {includePath}/tests/resources/trigger-fatal-exception.php(23) #0 [internal function]: {closure}('CoreHome', 'index', Array) #1 {includePath}/core/EventDispatcher.php(141): call_user_func_array(Object(Closure), Array) #2 {includePath}/core/Piwik.php(756): Piwik\EventDispatcher->postEvent('Request.dispatc...', Array, false, Array) #3 {includePath}/core/FrontController.php(569): Piwik\Piwik::postEvent('Request.dispatc...', Array) #4 {includePath}/core/FrontController.php(165): Piwik\FrontController->doDispatch('CoreHome', 'index', Array) #5 {includePath}/tests/resources/trigger-fatal-exception.php(31): Piwik\FrontController->dispatch('CoreHome', 'index') #6 {main} +test message on {includePath}/tests/resources/trigger-fatal-exception.php(23) #0 [internal function]: {closure}('CoreHome', 'index', Array) #1 {includePath}/core/EventDispatcher.php(141): call_user_func_array(Object(Closure), Array) #2 {includePath}/core/Piwik.php(756): Piwik\EventDispatcher->postEvent('Request.dispatc...', Array, false, Array) #3 {includePath}/core/FrontController.php(570): Piwik\Piwik::postEvent('Request.dispatc...', Array) #4 {includePath}/core/FrontController.php(165): Piwik\FrontController->doDispatch('CoreHome', 'index', Array) #5 {includePath}/tests/resources/trigger-fatal-exception.php(31): Piwik\FrontController->dispatch('CoreHome', 'index') #6 {main} FORMAT; $this->assertStringMatchesFormat($expectedFormat, $response['message']); diff --git a/tests/PHPUnit/Integration/Period/FactoryTest.php b/tests/PHPUnit/Integration/Period/FactoryTest.php index c83b231c82..7df6f71d36 100644 --- a/tests/PHPUnit/Integration/Period/FactoryTest.php +++ b/tests/PHPUnit/Integration/Period/FactoryTest.php @@ -83,6 +83,7 @@ class FactoryTest extends IntegrationTestCase ['range', '2015-01-01,2015-01-10', 'UTC', Range::class, '2015-01-01,2015-01-10'], ['range', '2015-01-01,2015-01-10', 'Antarctica/Casey', Range::class, '2015-01-01,2015-01-10'], + ['range', '2015-01-01,2015-01-01', 'Antarctica/Casey', Day::class, '2015-01-01,2015-01-01'], // multiple periods ['day', '2015-01-01,2015-01-10', 'UTC', Range::class, '2015-01-01,2015-01-10'], @@ -92,6 +93,13 @@ class FactoryTest extends IntegrationTestCase ]; } + public function test_makePeriodFromQueryParams() + { + $factory = Period\Factory::makePeriodFromQueryParams('UTC', 'range', '2019-01-01,2019-01-01'); + $this->assertTrue($factory instanceof Day); + $this->assertEquals('2019-01-01', $factory->toString()); + } + public function test_build_CreatesCustomPeriodInstances() { Config::getInstance()->General['enabled_periods_API'] .= ',customperiod'; diff --git a/tests/PHPUnit/Integration/SessionTest.php b/tests/PHPUnit/Integration/SessionTest.php index 67020e2835..4664611dfb 100644 --- a/tests/PHPUnit/Integration/SessionTest.php +++ b/tests/PHPUnit/Integration/SessionTest.php @@ -27,4 +27,46 @@ class SessionTest extends \PHPUnit\Framework\TestCase $this->assertSame('ok', trim($result)); } + /** + * @dataProvider getCookieTests + */ + public function testWriteCookie($expected, $name, $value, $expires, $path, $domain, $secure, $httpOnly, $sameSite) + { + $result = Session::writeCookie($name, $value, $expires, $path, $domain, $secure, $httpOnly, $sameSite); + $this->assertEquals($expected, $result); + } + + public function getCookieTests() + { + return [ + [ + 'Set-Cookie: myname=myvalue; expires=Tue, 03-May-2022 02:27:34 GMT; path=/; domain=my.test.domain; secure; httponly; SameSite=lax', + 'myname', 'myvalue', 1651544854, '/', 'my.test.domain', true, true, 'lax' + ], + [ + 'Set-Cookie: myname=myvalue; expires=Tue, 03-May-2022 02:27:34 GMT; path=/; domain=my.test.domain; httponly; SameSite=none', + 'myname', 'myvalue', 1651544854, '/', 'my.test.domain', false, true, 'none' + ], + [ + 'Set-Cookie: %3Cxss%3Emyname%26%24=my%3Cxss%3E%27%24%25value; expires=Tue, 03-May-2022 02:27:34 GMT; path=/; domain=ma%3Cf0r3%24%25%25.tld; SameSite=lax', + '<xss>myname&$', 'my<xss>\'$%value', 1651544854, '/', 'ma<f0r3$%%.tld', false, false, 'lax' + ], + [ + 'Set-Cookie: myname=myvalue; expires=Tue, 03-May-2022 02:27:34 GMT; path=/; domain=my.test.domain', + 'myname', 'myvalue', 1651544854, '/', 'my.test.domain', false, false, '' + ], + [ + 'Set-Cookie: myname=myvalue; expires=Tue, 03-May-2022 02:27:34 GMT; path=/', + 'myname', 'myvalue', 1651544854, '/', '', false, false, '' + ], + [ + 'Set-Cookie: myname=myvalue; path=/', + 'myname', 'myvalue', 0, '/', '', false, false, '' + ], + [ + 'Set-Cookie: myname=myvalue', + 'myname', 'myvalue', 0, '', '', false, false, '' + ], + ]; + } } diff --git a/tests/PHPUnit/Integration/Tracker/VisitTest.php b/tests/PHPUnit/Integration/Tracker/VisitTest.php index eb702586c5..8fb265fe9e 100644 --- a/tests/PHPUnit/Integration/Tracker/VisitTest.php +++ b/tests/PHPUnit/Integration/Tracker/VisitTest.php @@ -392,7 +392,7 @@ class VisitTest extends IntegrationTestCase $currentActionTime = Date::today()->getDatetime(); $idsite = API::getInstance()->addSite('name', 'http://piwik.net/'); - $expectedRemembered = array(Date::today()->toString() => [1]); + $expectedRemembered = array(); $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $currentActionTime, $expectedRemembered); } @@ -414,7 +414,6 @@ class VisitTest extends IntegrationTestCase // The double-handling below is needed to work around weird behaviour when UTC and UTC+5 are different dates // Example: 4:32am on 1 April in UTC+5 is 11:32pm on 31 March in UTC $midnight = Date::factoryInTimezone('today', 'UTC+5')->setTimezone('UTC+5'); - $today = Date::factoryInTimezone('today', 'UTC+5'); $oneHourAfterMidnight = $midnight->addHour(1)->getDatetime(); $oneHourBeforeMidnight = $midnight->subHour(1)->getDatetime(); @@ -428,11 +427,10 @@ class VisitTest extends IntegrationTestCase $expectedRemembered = array( substr($oneHourAfterMidnight, 0, 10) => array($idsite), - $today->toString() => [$idsite], ); // if website timezone was von considered both would be today (expected = array()) - $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $oneHourAfterMidnight, array($today->toString() => [$idsite])); + $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $oneHourAfterMidnight, array()); $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $oneHourBeforeMidnight, $expectedRemembered); } @@ -451,7 +449,22 @@ class VisitTest extends IntegrationTestCase $archive = StaticContainer::get('Piwik\Archive\ArchiveInvalidator'); $remembered = $archive->getRememberedArchivedReportsThatShouldBeInvalidated(); - $this->assertSame($expectedRemeberedArchivedReports, $remembered); + $this->assertSameReportsInvalidated($expectedRemeberedArchivedReports, $remembered); + } + + private function assertSameReportsInvalidated($expected, $actual) + { + $keys1 = array_keys($expected); + $keys2 = array_keys($actual); + sort($keys1); + sort($keys2); + + $this->assertSame($keys1, $keys2); + foreach ($expected as $index => $values) { + sort($values); + sort($actual[$index]); + $this->assertSame($values, $actual[$index]); + } } private function prepareVisitWithRequest($requestParams, $requestDate) diff --git a/tests/PHPUnit/Integration/TrackerTest.php b/tests/PHPUnit/Integration/TrackerTest.php index db7a644bbb..3b7786940e 100644 --- a/tests/PHPUnit/Integration/TrackerTest.php +++ b/tests/PHPUnit/Integration/TrackerTest.php @@ -369,7 +369,7 @@ class TrackerTest extends IntegrationTestCase public function test_archiveInvalidation_differentServerAndWebsiteTimezones() { // Server timezone is UTC - ini_set('date.timezone', 'America/New_York'); + ini_set('date.timezone', 'UTC'); // Website timezone is New York $idSite = Fixture::createWebsite('2014-01-01 00:00:00', 0, false, false, @@ -385,8 +385,8 @@ class TrackerTest extends IntegrationTestCase $this->request->setCurrentTimestamp(Date::$now); $this->tracker->trackRequest($this->request); - // make sure today archives are also invalidated - $this->assertEquals(['report_to_invalidate_2_2019-04-02_' . getmypid() => '1'], Option::getLike('report_to_invalidate_2_2019-04-02%')); + // make sure today archives are not invalidated + $this->assertEquals([], Option::getLike('report_to_invalidate_2_2019-04-02%')); } public function test_TrackingNewVisitOfKnownVisitor() diff --git a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php index 04fdbe8b61..f706896868 100644 --- a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php +++ b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest.php @@ -7,8 +7,11 @@ */ namespace Piwik\Tests\System; +use Piwik\Archive\ArchivePurger; use Piwik\Archive\Chunk; use Piwik\Common; +use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Db; use Piwik\Piwik; use Piwik\Tests\Framework\TestCase\SystemTestCase; @@ -23,6 +26,9 @@ use Piwik\Tests\Fixtures\VisitsOverSeveralDays; */ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCase { + /** + * @var VisitsOverSeveralDays + */ public static $fixture = null; // initialized below test definition public static function getOutputPrefix() @@ -101,6 +107,11 @@ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCa */ public function test_checkArchiveRecords_whenPeriodIsRange() { + $archivePurger = StaticContainer::get(ArchivePurger::class); + foreach (self::$fixture->dateTimes as $date) { + $archivePurger->purgeInvalidatedArchivesFrom(Date::factory($date)); + } + // we expect 6 blobs for Actions plugins, because flat=1 or expanded=1 was not set // so we only archived the parent table $expectedActionsBlobs = 6; diff --git a/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php b/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php index 22a8cca5f8..f4708492d9 100644 --- a/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php +++ b/tests/PHPUnit/System/OneVisitorTwoVisitsTest.php @@ -78,6 +78,7 @@ class OneVisitorTwoVisitsTest extends SystemTestCase foreach ($bulkUrls as &$url) { $url = urlencode($url); } + return array( array('all', array('idSite' => $idSite, 'date' => $dateTime, diff --git a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php index cf22ba95cb..1eadd1b9cb 100644 --- a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php +++ b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php @@ -8,8 +8,10 @@ namespace Piwik\Tests\System; use Piwik\Archive; +use Piwik\Archive\ArchivePurger; use Piwik\Cache; use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Segment; use Piwik\Tests\Framework\TestCase\SystemTestCase; use Piwik\Tests\Fixtures\TwoSitesTwoVisitorsDifferentDays; @@ -139,10 +141,13 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase ); } - // TODO: this test should be in an integration test for Piwik\Archive. setup code for getting metrics from different - // plugins is non-trivial, so not done now. public function test_Archive_getNumeric_shouldInvalidateRememberedReportsOncePerRequestIfNeeded() { + /* TODO: remove this test and replace w/ integration test for invalidation in archive.php workflow + + $archivePurger = StaticContainer::get(ArchivePurger::class); + $archivePurger->purgeInvalidatedArchivesFrom(Date::factory(self::$fixture->dateTime)); + // Tests that getting a visits summary metric (nb_visits) & a Goal's metric (Goal_revenue) // at the same time works. $dateTimeRange = '2010-01-03,2010-01-06'; @@ -160,10 +165,6 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase $result ); - $cache = Cache::getTransientCache(); - $this->assertEquals(array(self::$fixture->idSite1, self::$fixture->idSite2), - $cache->fetch('Archive.SiteIdsOfRememberedReportsInvalidated')); - $invalidator = StaticContainer::get('Piwik\Archive\ArchiveInvalidator'); self::$fixture->trackVisits(); @@ -187,9 +188,6 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase // make sure the caching in archive::get() worked and they are still to be invalidated $this->assertCount(10, $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); - // now we force to actually invalidate archived reports again and then archive will be rebuilt for requsted siteId = 1 - $cache->delete('Archive.SiteIdsOfRememberedReportsInvalidated'); - $archive = Archive::build($idSite1, 'range', $dateTimeRange); $result = $archive->getNumeric($columns); @@ -205,6 +203,7 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase ), $result ); + */ } public static function getOutputPrefix() diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php index 675fb409b9..75ba96d7de 100644 --- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php +++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php @@ -7,9 +7,12 @@ */ namespace Piwik\Tests\System; +use Piwik\Archive\ArchivePurger; use Piwik\Archive\Chunk; use Piwik\Common; use Piwik\Archive\ArchiveInvalidator; +use Piwik\Container\StaticContainer; +use Piwik\Date; use Piwik\Db; use Piwik\Tests\Framework\TestCase\SystemTestCase; use Piwik\Tests\Fixtures\TwoVisitsWithCustomVariables; @@ -64,6 +67,9 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest */ public function testCheck() { + $archivePurger = StaticContainer::get(ArchivePurger::class); + $archivePurger->purgeInvalidatedArchivesFrom(Date::factory(self::$fixture->dateTime)); + // ---------------------------------------------- // Implementation Checks // ---------------------------------------------- @@ -90,9 +96,10 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest 'archive_blob_2009_12' => 20, // 7 metrics, // 2 Referrer metrics (Referrers_distinctSearchEngines/Referrers_distinctKeywords), - // 6 done flag (referrers, CustomVar, VisitsSummary), 3 for period = 1 and 3 for period = 2 + // 5 done flag (referrers, VisitsSummary), 2 for period = 1 and 3 for period = 2 // X * 2 segments - 'archive_numeric_2009_12' => (6 + 2 + 3 + 3) * 2, + // + 1 done flag archive for CustomVar + 'archive_numeric_2009_12' => (5 + 2 + 3 + 3) * 2 + 1, ); foreach ($tests as $table => $expectedRows) { $sql = "SELECT count(*) FROM " . Common::prefixTable($table); diff --git a/tests/PHPUnit/Unit/Config/IniFileChainCacheTest.php b/tests/PHPUnit/Unit/Config/IniFileChainCacheTest.php index 9ab66d3c3a..a87fbca4a8 100644 --- a/tests/PHPUnit/Unit/Config/IniFileChainCacheTest.php +++ b/tests/PHPUnit/Unit/Config/IniFileChainCacheTest.php @@ -130,14 +130,18 @@ class IniFileChainCacheTest extends IniFileChainTest $value = $this->cache->doFetch(IniFileChain::CONFIG_CACHE_KEY); $this->assertEquals(false, $value); + $userSettingsFileCopy = dirname($userSettingsFile) . '/copy.' . basename($userSettingsFile); + copy($userSettingsFile, $userSettingsFileCopy); + // reading the chain should populate the cache - $fileChain = new TestIniFileChain($defaultSettingFiles, $userSettingsFile, $this->testHost); + $fileChain = new TestIniFileChain($defaultSettingFiles, $userSettingsFileCopy, $this->testHost); $expected['General'] = array('trusted_hosts' => array($this->testHost)); $this->assertEquals($expected, $fileChain->getAll(), "'$testDescription' failed"); // even though the passed config files don't exist it still returns the same result as it is fetched from // cache - $testChain = new TestIniFileChain(array('foo'), 'bar'); + unlink($userSettingsFileCopy); + $testChain = new TestIniFileChain(array('foo'), $userSettingsFileCopy); $this->assertEquals($expected, $testChain->getAll(), "'$testDescription' failed"); } diff --git a/tests/PHPUnit/Unit/DbHelperTest.php b/tests/PHPUnit/Unit/DbHelperTest.php index 28613de7b8..b085335bf8 100644 --- a/tests/PHPUnit/Unit/DbHelperTest.php +++ b/tests/PHPUnit/Unit/DbHelperTest.php @@ -15,6 +15,7 @@ use Piwik\DbHelper; * @package Piwik\Tests\Unit * @group Core * @group Core_Unit + * @group DbHelper */ class DbHelperTest extends \PHPUnit\Framework\TestCase { @@ -58,4 +59,23 @@ class DbHelperTest extends \PHPUnit\Framework\TestCase ), ); } + + /** + * @dataProvider getTestQueries + */ + public function testAddMaxExecutionTimeHintToQuery($expected, $query, $timeLimit) + { + $result = DbHelper::addMaxExecutionTimeHintToQuery($query, $timeLimit); + $this->assertEquals($expected, $result); + } + + public function getTestQueries() + { + return [ + ['SELECT /*+ MAX_EXECUTION_TIME(1500) */ * FROM table', 'SELECT * FROM table', 1.5], + ['SELECT /*+ MAX_EXECUTION_TIME(20000) */ column FROM (SELECT * FROM table)', 'SELECT column FROM (SELECT * FROM table)', 20], + ['SELECT * FROM table', 'SELECT * FROM table', 0], + ['UPDATE table SET column = value', 'UPDATE table SET column = value', 150], + ]; + } } diff --git a/tests/PHPUnit/Unit/PeriodTest.php b/tests/PHPUnit/Unit/PeriodTest.php index 2d90fdbfdc..6e4f16f7b2 100644 --- a/tests/PHPUnit/Unit/PeriodTest.php +++ b/tests/PHPUnit/Unit/PeriodTest.php @@ -255,4 +255,29 @@ class PeriodTest extends \PHPUnit\Framework\TestCase return array($period->getLabel(), $period->getRangeString()); }, $periods); } + + /** + * @dataProvider getTestDataForIsDateInPeriod + */ + public function test_isDateInPeriod($date, $period, $periodDate, $expected) + { + $date = Date::factory($date); + $period = Period\Factory::build($period, $periodDate); + + $actual = $period->isDateInPeriod($date); + $this->assertEquals($expected, $actual); + } + + public function getTestDataForIsDateInPeriod() + { + return [ + ['2014-02-03 00:00:00', 'day', '2014-02-03 03:04:05', true], + ['2014-02-03 00:00:00', 'week', '2014-02-03 03:04:05', true], + ['2014-02-03 00:00:00', 'month', '2014-02-03 03:04:05', true], + ['2014-02-02 23:59:59', 'day', '2014-02-03 03:04:05', false], + ['2014-01-31 23:59:59', 'month', '2014-02-03 03:04:05', false], + ['2014-03-01 00:00:00', 'month', '2014-02-03 03:04:05', false], + ['2014-03-31 23:59:59', 'month', '2014-03-03 03:04:05', true], + ]; + } }
\ No newline at end of file diff --git a/tests/UI/expected-screenshots/OptOutForm_clicked_once.png b/tests/UI/expected-screenshots/OptOutForm_clicked_once.png new file mode 100644 index 0000000000..b349e78513 --- /dev/null +++ b/tests/UI/expected-screenshots/OptOutForm_clicked_once.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c2c658d66186e231303940ef50ba4a8cc982d28c0425e4290cb10ea3ea4e46d0 +size 22990 diff --git a/tests/UI/expected-screenshots/OptOutForm_clicked_twice.png b/tests/UI/expected-screenshots/OptOutForm_clicked_twice.png new file mode 100644 index 0000000000..152f97f608 --- /dev/null +++ b/tests/UI/expected-screenshots/OptOutForm_clicked_twice.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:375f8b477c06ee4477d7976f347dae645ea04a97c0acd21b9f154bd61edfc71b +size 23291 diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_goals_individual_goal.png b/tests/UI/expected-screenshots/UIIntegrationTest_goals_individual_goal.png index b35029c4ab..9592492741 100644 --- a/tests/UI/expected-screenshots/UIIntegrationTest_goals_individual_goal.png +++ b/tests/UI/expected-screenshots/UIIntegrationTest_goals_individual_goal.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c2198a7973f8b50aec5a79390dc504b1bc90740e5b288c9d6188e26043b11bfd -size 196431 +oid sha256:aae12be978bce7c4e98410b126aa8772c5010c9073117c5c3d4b9be9353f2c47 +size 196043 diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_goals_individual_goal_updated.png b/tests/UI/expected-screenshots/UIIntegrationTest_goals_individual_goal_updated.png index 6463823f7d..f08d6937ec 100644 --- a/tests/UI/expected-screenshots/UIIntegrationTest_goals_individual_goal_updated.png +++ b/tests/UI/expected-screenshots/UIIntegrationTest_goals_individual_goal_updated.png @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:9391cc9f9938c822d70ba16644b1efc3fc4b2ff220f1fb3d7810f9d076b302fd -size 197279 +oid sha256:92c77f3c5e7ad2578f3864ca1c839b0b930a91b873c28982da7a32e258f50eb7 +size 196734 diff --git a/tests/UI/specs/OptOutForm_spec.js b/tests/UI/specs/OptOutForm_spec.js index 484ac2146f..97f64d2885 100644 --- a/tests/UI/specs/OptOutForm_spec.js +++ b/tests/UI/specs/OptOutForm_spec.js @@ -61,6 +61,37 @@ describe("OptOutForm", function () { expect(await element.screenshot()).to.matchImage('opted-out_reloaded'); }); + it("using opt out twice should work correctly", async function () { + page.setUserAgent(chromeUserAgent); + await page.goto(siteUrl); + + await page.evaluate(function () { + $('iframe#optOutIframe').contents().find('input#trackVisits').click(); + }); + + await page.waitFor(5000); + + await expandIframe(); + + // check the box has opted in state after clicking once + var element = await page.jQuery('iframe#optOutIframe'); + expect(await element.screenshot()).to.matchImage('clicked_once'); + + await page.evaluate(function () { + $('iframe#optOutIframe').contents().find('input#trackVisits').click(); + }); + + await page.waitFor(5000); + + // check the box has outed out state after click another time + await page.reload(); + + await expandIframe(); + + var element = await page.jQuery('iframe#optOutIframe'); + expect(await element.screenshot()).to.matchImage('clicked_twice'); + }); + it("should correctly show display opted-in form when cookies are cleared", async function () { await page.clearCookies(); |
