diff options
author | diosmosis <benaka@piwik.pro> | 2015-03-15 22:31:25 +0300 |
---|---|---|
committer | diosmosis <benaka@piwik.pro> | 2015-03-15 22:31:25 +0300 |
commit | 2c325c6352619fe7967233d78860e41b35813a7a (patch) | |
tree | 6fd68b56bf910fdcbb463b7adc39bd9fb90cdf47 /core/DataAccess | |
parent | d03cae1e4c5113f987c1df3ab90ab6e0e9f4e624 (diff) | |
parent | 649337c7a37a3fda9a35532240ef06092fd2f7e3 (diff) |
Merge branch 'master' into geo-attribution-task
Diffstat (limited to 'core/DataAccess')
-rw-r--r-- | core/DataAccess/ArchiveInvalidator.php | 330 | ||||
-rw-r--r-- | core/DataAccess/ArchivePurger.php | 139 | ||||
-rw-r--r-- | core/DataAccess/InvalidatedReports.php | 168 | ||||
-rw-r--r-- | core/DataAccess/Model.php | 17 |
4 files changed, 17 insertions, 637 deletions
diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/DataAccess/ArchiveInvalidator.php deleted file mode 100644 index 916d94991b..0000000000 --- a/core/DataAccess/ArchiveInvalidator.php +++ /dev/null @@ -1,330 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - */ - -namespace Piwik\DataAccess; - -use Piwik\Date; -use Piwik\Db; -use Piwik\Option; -use Piwik\Plugins\PrivacyManager\PrivacyManager; -use Piwik\Period; -use Piwik\Period\Week; -use Piwik\Plugins\SitesManager\Model as SitesManagerModel; -use Piwik\Site; - -/** - * Marks archives as Invalidated by setting the done flag to a special value (see Model->updateArchiveAsInvalidated) - * - * Invalidated archives can still be selected and displayed in UI and API (until they are reprocessed by core:archive) - * - * The invalidated archives will be deleted by ArchivePurger - * - * @package Piwik\DataAccess - */ -class ArchiveInvalidator { - - private $warningDates = array(); - private $processedDates = array(); - private $minimumDateWithLogs = false; - private $invalidDates = array(); - - private $rememberArchivedReportIdStart = 'report_to_invalidate_'; - - public function rememberToInvalidateArchivedReportsLater($idSite, Date $date) - { - $key = $this->buildRememberArchivedReportId($idSite, $date->toString()); - $value = Option::get($key); - - // we do not really have to get the value first. we could simply always try to call set() and it would update or - // insert the record if needed but we do not want to lock the table (especially since there are still some - // MyISAM installations) - - if (false === $value) { - Option::set($key, '1'); - } - } - - public function getRememberedArchivedReportsThatShouldBeInvalidated() - { - $reports = Option::getLike($this->rememberArchivedReportIdStart . '%_%'); - - $sitesPerDay = array(); - - foreach ($reports as $report => $value) { - $report = str_replace($this->rememberArchivedReportIdStart, '', $report); - $report = explode('_', $report); - $siteId = (int) $report[0]; - $date = $report[1]; - - if (empty($sitesPerDay[$date])) { - $sitesPerDay[$date] = array(); - } - - $sitesPerDay[$date][] = $siteId; - } - - return $sitesPerDay; - } - - private function buildRememberArchivedReportId($idSite, $date) - { - $id = $this->buildRememberArchivedReportIdForSite($idSite); - $id .= '_' . trim($date); - - return $id; - } - - private function buildRememberArchivedReportIdForSite($idSite) - { - return $this->rememberArchivedReportIdStart . (int) $idSite; - } - - public function forgetRememberedArchivedReportsToInvalidateForSite($idSite) - { - $id = $this->buildRememberArchivedReportIdForSite($idSite) . '_%'; - Option::deleteLike($id); - } - - /** - * @internal - */ - public function forgetRememberedArchivedReportsToInvalidate($idSite, Date $date) - { - $id = $this->buildRememberArchivedReportId($idSite, $date->toString()); - - Option::delete($id); - } - - /** - * @param $idSites array - * @param $dates string - * @param $period string - * @return array - * @throws \Exception - */ - public function markArchivesAsInvalidated(array $idSites, $dates, $period) - { - $this->findOlderDateWithLogs(); - $datesToInvalidate = $this->getDatesToInvalidateFromString($dates); - $minDate = $this->getMinimumDateToInvalidate($datesToInvalidate); - - $this->updateSiteCreatedTime($idSites, $minDate); - - $datesByMonth = $this->getDatesByYearMonth($datesToInvalidate); - $this->markArchivesInvalidatedFor($idSites, $period, $datesByMonth); - - $this->persistInvalidatedArchives($idSites, $datesByMonth); - - foreach ($idSites as $idSite) { - foreach ($datesToInvalidate as $date) { - $this->forgetRememberedArchivedReportsToInvalidate($idSite, $date); - } - } - - return $this->makeOutputLogs(); - } - - private function updateSiteCreatedTime($idSites, Date $minDate) - { - $idSites = Site::getIdSitesFromIdSitesString($idSites); - $minDateSql = $minDate->subDay(1)->getDatetime(); - - $model = new SitesManagerModel(); - $model->updateSiteCreatedTime($idSites, $minDateSql); - } - - /** - * @param $toInvalidate - * @return bool|Date - * @throws \Exception - */ - private function getMinimumDateToInvalidate($toInvalidate) - { - /* @var $date Date */ - $minDate = false; - foreach ($toInvalidate as $date) { - // Keep track of the minimum date for each website - if ($minDate === false - || $date->isEarlier($minDate) - ) { - $minDate = $date; - } - } - if (empty($minDate)) { - throw new \Exception("Check the 'dates' parameter is a valid date."); - } - return $minDate; - } - - /** - * @param $idSites - * @param $period string - * @param $datesByMonth array - * @throws \Exception - */ - private function markArchivesInvalidatedFor($idSites, $period, $datesByMonth) - { - $invalidateForPeriodId = $this->getPeriodId($period); - - // In each table, invalidate day/week/month/year containing this date - $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled(); - - $archiveNumericTables = array_filter($archiveTables, function($name) { - return ArchiveTableCreator::getTypeFromTableName($name) == ArchiveTableCreator::NUMERIC_TABLE; - }); - - foreach ($archiveNumericTables as $table) { - // Extract Y_m from table name - $suffix = ArchiveTableCreator::getDateFromTableName($table); - if (!isset($datesByMonth[$suffix])) { - continue; - } - // Dates which are to be deleted from this table - $datesToDelete = $datesByMonth[$suffix]; - self::getModel()->updateArchiveAsInvalidated($table, $idSites, $invalidateForPeriodId, $datesToDelete); - } - } - - /** - * Ensure the specified dates are valid. - * Store invalid date so we can log them - * @param array $dates - * @return Date[] - */ - private function getDatesToInvalidateFromString($dates) - { - $toInvalidate = array(); - - $dates = explode(',', trim($dates)); - $dates = array_unique($dates); - - foreach ($dates as $theDate) { - $theDate = trim($theDate); - try { - $date = Date::factory($theDate); - } catch (\Exception $e) { - $this->invalidDates[] = $theDate; - continue; - } - if ($date->toString() == $theDate) { - $toInvalidate[] = $date; - } else { - $this->invalidDates[] = $theDate; - } - } - - return $toInvalidate; - } - - private function findOlderDateWithLogs() - { - // If using the feature "Delete logs older than N days"... - $purgeDataSettings = PrivacyManager::getPurgeDataSettings(); - $logsDeletedWhenOlderThanDays = $purgeDataSettings['delete_logs_older_than']; - $logsDeleteEnabled = $purgeDataSettings['delete_logs_enable']; - - if ($logsDeleteEnabled - && $logsDeletedWhenOlderThanDays - ) { - $this->minimumDateWithLogs = Date::factory('today')->subDay($logsDeletedWhenOlderThanDays); - } - } - - /** - * Given the list of dates, process which tables YYYY_MM we should delete from - * - * @param $datesToInvalidate Date[] - * @return array - */ - private function getDatesByYearMonth($datesToInvalidate) - { - $datesByMonth = array(); - foreach ($datesToInvalidate as $date) { - // we should only delete reports for dates that are more recent than N days - if ($this->minimumDateWithLogs - && $date->isEarlier($this->minimumDateWithLogs) - ) { - $this->warningDates[] = $date->toString(); - continue; - } - - $this->processedDates[] = $date->toString(); - - $month = $date->toString('Y_m'); - // For a given date, we must invalidate in the monthly archive table - $datesByMonth[$month][] = $date->toString(); - - // But also the year stored in January - $year = $date->toString('Y_01'); - $datesByMonth[$year][] = $date->toString(); - - // but also weeks overlapping several months stored in the month where the week is starting - /* @var $week Week */ - $week = Period\Factory::build('week', $date); - $weekAsString = $week->getDateStart()->toString('Y_m'); - $datesByMonth[$weekAsString][] = $date->toString(); - - } - return $datesByMonth; - } - - /** - * @return array - */ - private function makeOutputLogs() - { - $output = array(); - if ($this->warningDates) { - $output[] = 'Warning: the following Dates have not been invalidated, because they are earlier than your Log Deletion limit: ' . - implode(", ", $this->warningDates) . - "\n The last day with logs is " . $this->minimumDateWithLogs . ". " . - "\n Please disable 'Delete old Logs' or set it to a higher deletion threshold (eg. 180 days or 365 years).'."; - } - if ($this->invalidDates) { - $output[] = 'Warning: some of the Dates to invalidate were invalid: ' . - implode(", ", $this->invalidDates) . ". Piwik simply ignored those and proceeded with the others."; - } - - $output[] = "Success. The following dates were invalidated successfully: " . implode(", ", $this->processedDates); - return $output; - } - - /** - * @param $period - * @return bool|int - */ - private function getPeriodId($period) - { - if (!empty($period)) { - $period = Period\Factory::build($period, Date::today()); - } - $invalidateForPeriod = $period ? $period->getId() : false; - return $invalidateForPeriod; - } - - /** - * @param array $idSites - * @param $datesByMonth - */ - private function persistInvalidatedArchives(array $idSites, $datesByMonth) - { - $yearMonths = array_keys($datesByMonth); - $yearMonths = array_unique($yearMonths); - - $store = new InvalidatedReports(); - $store->addInvalidatedSitesToReprocess($idSites); - $store->addSitesToPurgeForYearMonths($idSites, $yearMonths); - } - - private static function getModel() - { - return new Model(); - } - -}
\ No newline at end of file diff --git a/core/DataAccess/ArchivePurger.php b/core/DataAccess/ArchivePurger.php deleted file mode 100644 index e7f185f475..0000000000 --- a/core/DataAccess/ArchivePurger.php +++ /dev/null @@ -1,139 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - */ -namespace Piwik\DataAccess; - -use Exception; -use Piwik\ArchiveProcessor\Rules; -use Piwik\Config; -use Piwik\Date; -use Piwik\Db; -use Piwik\Log; -use Piwik\Piwik; - -/** - * - * This class purges two types of archives: - * - * (1) Deletes invalidated archives (from ArchiveInvalidator) - * - * (2) Deletes outdated archives (the temporary or errored archives) - * - * - * @package Piwik\DataAccess - */ -class ArchivePurger -{ - public static function purgeInvalidatedArchives() - { - $store = new InvalidatedReports(); - $idSitesByYearMonth = $store->getSitesByYearMonthArchiveToPurge(); - foreach ($idSitesByYearMonth as $yearMonth => $idSites) { - if(empty($idSites)) { - continue; - } - - $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01'); - $numericTable = ArchiveTableCreator::getNumericTable($date); - - $archiveIds = self::getModel()->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites); - - if (count($archiveIds) == 0) { - continue; - } - self::deleteArchiveIds($date, $archiveIds); - - $store->markSiteIdsHaveBeenPurged($idSites, $yearMonth); - } - } - - /** - * Removes the outdated archives for the given month. - * (meaning they are marked with a done flag of ArchiveWriter::DONE_OK_TEMPORARY or ArchiveWriter::DONE_ERROR) - * - * @param Date $dateStart Only the month will be used - */ - public static function purgeOutdatedArchives(Date $dateStart) - { - $purgeArchivesOlderThan = Rules::shouldPurgeOutdatedArchives($dateStart); - - if (!$purgeArchivesOlderThan) { - return; - } - - $idArchivesToDelete = self::getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan); - - if (!empty($idArchivesToDelete)) { - self::deleteArchiveIds($dateStart, $idArchivesToDelete); - } - - self::deleteArchivesWithPeriodRange($dateStart); - - Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]", - $purgeArchivesOlderThan, - $dateStart->toString("Y-m"), - implode(',', $idArchivesToDelete)); - } - - protected static function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan) - { - $archiveTable = ArchiveTableCreator::getNumericTable($date); - - $result = self::getModel()->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan); - - $idArchivesToDelete = array(); - if (!empty($result)) { - foreach ($result as $row) { - $idArchivesToDelete[] = $row['idarchive']; - } - } - - return $idArchivesToDelete; - } - - /** - * Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space. - * - * @param $date Date - */ - protected static function deleteArchivesWithPeriodRange(Date $date) - { - $numericTable = ArchiveTableCreator::getNumericTable($date); - $blobTable = ArchiveTableCreator::getBlobTable($date); - $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days']; - $pastDate = Date::factory('today')->subDay($daysRangesValid)->getDateTime(); - - self::getModel()->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $pastDate); - - Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]", - $pastDate, $numericTable); - } - - /** - * Deletes by batches Archive IDs in the specified month, - * - * @param Date $date - * @param $idArchivesToDelete - */ - protected static function deleteArchiveIds(Date $date, $idArchivesToDelete) - { - $batches = array_chunk($idArchivesToDelete, 1000); - $numericTable = ArchiveTableCreator::getNumericTable($date); - $blobTable = ArchiveTableCreator::getBlobTable($date); - - foreach ($batches as $idsToDelete) { - self::getModel()->deleteArchiveIds($numericTable, $blobTable, $idsToDelete); - } - } - - private static function getModel() - { - return new Model(); - } - -} diff --git a/core/DataAccess/InvalidatedReports.php b/core/DataAccess/InvalidatedReports.php deleted file mode 100644 index 64f863e0ad..0000000000 --- a/core/DataAccess/InvalidatedReports.php +++ /dev/null @@ -1,168 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - */ - -namespace Piwik\DataAccess; - -use Piwik\Option; - -/** - * Keeps track of which reports were invalidated via CoreAdminHome.invalidateArchivedReports API. - * - * This is used by: - * - * 1. core:archive command to know which websites should be reprocessed - * - * 2. scheduled task purgeInvalidatedArchives to know which websites/months should be purged - * - */ -class InvalidatedReports -{ - const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds'; - const OPTION_INVALIDATED_DATES_SITES_TO_PURGE = 'InvalidatedOldReports_DatesWebsiteIds'; - - /** - * Mark the sites IDs and Dates as being invalidated, so we can purge them later on. - * - * @param array $idSites - * @param array $yearMonths - */ - public function addSitesToPurgeForYearMonths(array $idSites, $yearMonths) - { - $idSitesByYearMonth = $this->getSitesByYearMonthToPurge(); - - foreach($yearMonths as $yearMonthToPurge) { - - if(isset($idSitesByYearMonth[$yearMonthToPurge])) { - $existingIdSitesToPurge = $idSitesByYearMonth[$yearMonthToPurge]; - $idSites = array_merge($existingIdSitesToPurge, $idSites); - $idSites = array_unique($idSites); - } - $idSitesByYearMonth[$yearMonthToPurge] = $idSites; - } - $this->persistSitesByYearMonthToPurge($idSitesByYearMonth); - } - - /** - * Returns the list of websites IDs for which invalidated archives can be purged. - */ - public function getSitesByYearMonthArchiveToPurge() - { - $idSitesByYearMonth = $this->getSitesByYearMonthToPurge(); - - // From this list we remove the websites that are not yet re-processed - // so we don't purge them before they were re-processed - $idSitesNotYetReprocessed = $this->getSitesToReprocess(); - - foreach($idSitesByYearMonth as $yearMonth => &$idSites) { - $idSites = array_diff($idSites, $idSitesNotYetReprocessed); - } - return $idSitesByYearMonth; - } - - public function markSiteIdsHaveBeenPurged(array $idSites, $yearMonth) - { - $idSitesByYearMonth = $this->getSitesByYearMonthToPurge(); - - if(!isset($idSitesByYearMonth[$yearMonth])) { - return; - } - - $idSitesByYearMonth[$yearMonth] = array_diff($idSitesByYearMonth[$yearMonth], $idSites); - $this->persistSitesByYearMonthToPurge($idSitesByYearMonth); - } - - /** - * Record those website IDs as having been invalidated - * - * @param $idSites - */ - public function addInvalidatedSitesToReprocess(array $idSites) - { - $siteIdsToReprocess = $this->getSitesToReprocess(); - $siteIdsToReprocess = array_merge($siteIdsToReprocess, $idSites); - $this->setSitesToReprocess($siteIdsToReprocess); - } - - - /** - * @param $idSite - */ - public function storeSiteIsReprocessed($idSite) - { - $siteIdsToReprocess = $this->getSitesToReprocess(); - - if (count($siteIdsToReprocess)) { - $found = array_search($idSite, $siteIdsToReprocess); - if ($found !== false) { - unset($siteIdsToReprocess[$found]); - $this->setSitesToReprocess($siteIdsToReprocess); - } - } - } - - /** - * Returns array of idSites to force re-process next time core:archive command runs - * - * @return array of id sites - */ - public function getSitesToReprocess() - { - return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS); - } - - /** - * @return array|false|mixed|string - */ - private function getSitesByYearMonthToPurge() - { - return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE); - } - - /** - * @param $websiteIdsInvalidated - */ - private function setSitesToReprocess($websiteIdsInvalidated) - { - $websiteIdsInvalidated = array_unique($websiteIdsInvalidated); - $websiteIdsInvalidated = array_values($websiteIdsInvalidated); - Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated)); - } - - /** - * @param $optionName - * @return array|false|mixed|string - */ - private function getArrayValueFromOptionName($optionName) - { - Option::clearCachedOption($optionName); - $array = Option::get($optionName); - - if ($array - && ($array = unserialize($array)) - && count($array) - ) { - return $array; - } - return array(); - } - - /** - * @param $idSitesByYearMonth - */ - private function persistSitesByYearMonthToPurge($idSitesByYearMonth) - { - // remove dates for which there are no sites to purge - $idSitesByYearMonth = array_filter($idSitesByYearMonth); - - Option::set(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE, serialize($idSitesByYearMonth)); - } - - - -}
\ No newline at end of file diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index 239bce6a23..0980983fe3 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -246,6 +246,23 @@ class Model } /** + * Returns the site IDs for invalidated archives in an archive table. + * + * @param string $numericTable The numeric table to search through. + * @return int[] + */ + public function getSitesWithInvalidatedArchive($numericTable) + { + $rows = Db::fetchAll("SELECT DISTINCT idsite FROM `$numericTable` WHERE name LIKE 'done%' AND value = " . ArchiveWriter::DONE_INVALIDATED); + + $result = array(); + foreach ($rows as $row) { + $result[] = $row['idsite']; + } + return $result; + } + + /** * Returns the SQL condition used to find successfully completed archives that * this instance is querying for. */ |