Merge branch 'master' into geo-attribution-task

4 files changed, 17 insertions, 637 deletions

diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/DataAccess/ArchiveInvalidator.php
deleted file mode 100644
index 916d94991b..0000000000
--- a/core/DataAccess/ArchiveInvalidator.php
+++ /dev/null
@@ -1,330 +0,0 @@
-<?php
-/**
- * Piwik - free/libre analytics platform
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- *
- */
-
-namespace Piwik\DataAccess;
-
-use Piwik\Date;
-use Piwik\Db;
-use Piwik\Option;
-use Piwik\Plugins\PrivacyManager\PrivacyManager;
-use Piwik\Period;
-use Piwik\Period\Week;
-use Piwik\Plugins\SitesManager\Model as SitesManagerModel;
-use Piwik\Site;
-
-/**
- * Marks archives as Invalidated by setting the done flag to a special value (see Model->updateArchiveAsInvalidated)
- *
- * Invalidated archives can still be selected and displayed in UI and API (until they are reprocessed by core:archive)
- *
- * The invalidated archives will be deleted by ArchivePurger
- *
- * @package Piwik\DataAccess
- */
-class ArchiveInvalidator {
-
-    private $warningDates = array();
-    private $processedDates = array();
-    private $minimumDateWithLogs = false;
-    private $invalidDates = array();
-
-    private $rememberArchivedReportIdStart = 'report_to_invalidate_';
-
-    public function rememberToInvalidateArchivedReportsLater($idSite, Date $date)
-    {
-        $key   = $this->buildRememberArchivedReportId($idSite, $date->toString());
-        $value = Option::get($key);
-
-        // we do not really have to get the value first. we could simply always try to call set() and it would update or
-        // insert the record if needed but we do not want to lock the table (especially since there are still some
-        // MyISAM installations)
-
-        if (false === $value) {
-            Option::set($key, '1');
-        }
-    }
-
-    public function getRememberedArchivedReportsThatShouldBeInvalidated()
-    {
-        $reports = Option::getLike($this->rememberArchivedReportIdStart . '%_%');
-
-        $sitesPerDay = array();
-
-        foreach ($reports as $report => $value) {
-            $report = str_replace($this->rememberArchivedReportIdStart, '', $report);
-            $report = explode('_', $report);
-            $siteId = (int) $report[0];
-            $date   = $report[1];
-
-            if (empty($sitesPerDay[$date])) {
-                $sitesPerDay[$date] = array();
-            }
-
-            $sitesPerDay[$date][] = $siteId;
-        }
-
-        return $sitesPerDay;
-    }
-
-    private function buildRememberArchivedReportId($idSite, $date)
-    {
-        $id  = $this->buildRememberArchivedReportIdForSite($idSite);
-        $id .= '_' . trim($date);
-
-        return $id;
-    }
-
-    private function buildRememberArchivedReportIdForSite($idSite)
-    {
-        return $this->rememberArchivedReportIdStart . (int) $idSite;
-    }
-
-    public function forgetRememberedArchivedReportsToInvalidateForSite($idSite)
-    {
-        $id = $this->buildRememberArchivedReportIdForSite($idSite) . '_%';
-        Option::deleteLike($id);
-    }
-
-    /**
-     * @internal
-     */
-    public function forgetRememberedArchivedReportsToInvalidate($idSite, Date $date)
-    {
-        $id = $this->buildRememberArchivedReportId($idSite, $date->toString());
-
-        Option::delete($id);
-    }
-
-    /**
-     * @param $idSites array
-     * @param $dates string
-     * @param $period string
-     * @return array
-     * @throws \Exception
-     */
-    public function markArchivesAsInvalidated(array $idSites, $dates, $period)
-    {
-        $this->findOlderDateWithLogs();
-        $datesToInvalidate = $this->getDatesToInvalidateFromString($dates);
-        $minDate = $this->getMinimumDateToInvalidate($datesToInvalidate);
-
-        $this->updateSiteCreatedTime($idSites, $minDate);
-
-        $datesByMonth = $this->getDatesByYearMonth($datesToInvalidate);
-        $this->markArchivesInvalidatedFor($idSites, $period, $datesByMonth);
-
-        $this->persistInvalidatedArchives($idSites, $datesByMonth);
-
-        foreach ($idSites as $idSite) {
-            foreach ($datesToInvalidate as $date) {
-                $this->forgetRememberedArchivedReportsToInvalidate($idSite, $date);
-            }
-        }
-
-        return $this->makeOutputLogs();
-    }
-
-    private function updateSiteCreatedTime($idSites, Date $minDate)
-    {
-        $idSites    = Site::getIdSitesFromIdSitesString($idSites);
-        $minDateSql = $minDate->subDay(1)->getDatetime();
-
-        $model = new SitesManagerModel();
-        $model->updateSiteCreatedTime($idSites, $minDateSql);
-    }
-
-    /**
-     * @param $toInvalidate
-     * @return bool|Date
-     * @throws \Exception
-     */
-    private function getMinimumDateToInvalidate($toInvalidate)
-    {
-        /* @var $date Date */
-        $minDate = false;
-        foreach ($toInvalidate as $date) {
-            // Keep track of the minimum date for each website
-            if ($minDate === false
-                || $date->isEarlier($minDate)
-            ) {
-                $minDate = $date;
-            }
-        }
-        if (empty($minDate)) {
-            throw new \Exception("Check the 'dates' parameter is a valid date.");
-        }
-        return $minDate;
-    }
-
-    /**
-     * @param $idSites
-     * @param $period string
-     * @param $datesByMonth array
-     * @throws \Exception
-     */
-    private function markArchivesInvalidatedFor($idSites, $period, $datesByMonth)
-    {
-        $invalidateForPeriodId = $this->getPeriodId($period);
-
-        // In each table, invalidate day/week/month/year containing this date
-        $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
-
-        $archiveNumericTables = array_filter($archiveTables, function($name) {
-            return ArchiveTableCreator::getTypeFromTableName($name) == ArchiveTableCreator::NUMERIC_TABLE;
-        });
-
-        foreach ($archiveNumericTables as $table) {
-            // Extract Y_m from table name
-            $suffix = ArchiveTableCreator::getDateFromTableName($table);
-            if (!isset($datesByMonth[$suffix])) {
-                continue;
-            }
-            // Dates which are to be deleted from this table
-            $datesToDelete = $datesByMonth[$suffix];
-            self::getModel()->updateArchiveAsInvalidated($table, $idSites, $invalidateForPeriodId, $datesToDelete);
-        }
-    }
-
-    /**
-     * Ensure the specified dates are valid.
-     * Store invalid date so we can log them
-     * @param array $dates
-     * @return Date[]
-     */
-    private function getDatesToInvalidateFromString($dates)
-    {
-        $toInvalidate = array();
-
-        $dates = explode(',', trim($dates));
-        $dates = array_unique($dates);
-
-        foreach ($dates as $theDate) {
-            $theDate = trim($theDate);
-            try {
-                $date = Date::factory($theDate);
-            } catch (\Exception $e) {
-                $this->invalidDates[] = $theDate;
-                continue;
-            }
-            if ($date->toString() == $theDate) {
-                $toInvalidate[] = $date;
-            } else {
-                $this->invalidDates[] = $theDate;
-            }
-        }
-
-        return $toInvalidate;
-    }
-
-    private function findOlderDateWithLogs()
-    {
-        // If using the feature "Delete logs older than N days"...
-        $purgeDataSettings = PrivacyManager::getPurgeDataSettings();
-        $logsDeletedWhenOlderThanDays = $purgeDataSettings['delete_logs_older_than'];
-        $logsDeleteEnabled = $purgeDataSettings['delete_logs_enable'];
-
-        if ($logsDeleteEnabled
-            && $logsDeletedWhenOlderThanDays
-        ) {
-            $this->minimumDateWithLogs = Date::factory('today')->subDay($logsDeletedWhenOlderThanDays);
-        }
-    }
-
-    /**
-     * Given the list of dates, process which tables YYYY_MM we should delete from
-     *
-     * @param $datesToInvalidate Date[]
-     * @return array
-     */
-    private function getDatesByYearMonth($datesToInvalidate)
-    {
-        $datesByMonth = array();
-        foreach ($datesToInvalidate as $date) {
-            // we should only delete reports for dates that are more recent than N days
-            if ($this->minimumDateWithLogs
-                && $date->isEarlier($this->minimumDateWithLogs)
-            ) {
-                $this->warningDates[] = $date->toString();
-                continue;
-            }
-
-            $this->processedDates[] = $date->toString();
-
-            $month = $date->toString('Y_m');
-            // For a given date, we must invalidate in the monthly archive table
-            $datesByMonth[$month][] = $date->toString();
-
-            // But also the year stored in January
-            $year = $date->toString('Y_01');
-            $datesByMonth[$year][] = $date->toString();
-
-            // but also weeks overlapping several months stored in the month where the week is starting
-            /* @var $week Week */
-            $week = Period\Factory::build('week', $date);
-            $weekAsString = $week->getDateStart()->toString('Y_m');
-            $datesByMonth[$weekAsString][] = $date->toString();
-
-        }
-        return $datesByMonth;
-    }
-
-    /**
-     * @return array
-     */
-    private function makeOutputLogs()
-    {
-        $output = array();
-        if ($this->warningDates) {
-            $output[] = 'Warning: the following Dates have not been invalidated, because they are earlier than your Log Deletion limit: ' .
-                implode(", ", $this->warningDates) .
-                "\n The last day with logs is " . $this->minimumDateWithLogs . ". " .
-                "\n Please disable 'Delete old Logs' or set it to a higher deletion threshold (eg. 180 days or 365 years).'.";
-        }
-        if ($this->invalidDates) {
-            $output[] = 'Warning: some of the Dates to invalidate were invalid: ' .
-                implode(", ", $this->invalidDates) . ". Piwik simply ignored those and proceeded with the others.";
-        }
-
-        $output[] = "Success. The following dates were invalidated successfully: " . implode(", ", $this->processedDates);
-        return $output;
-    }
-
-    /**
-     * @param $period
-     * @return bool|int
-     */
-    private function getPeriodId($period)
-    {
-        if (!empty($period)) {
-            $period = Period\Factory::build($period, Date::today());
-        }
-        $invalidateForPeriod = $period ? $period->getId() : false;
-        return $invalidateForPeriod;
-    }
-
-    /**
-     * @param array $idSites
-     * @param $datesByMonth
-     */
-    private function persistInvalidatedArchives(array $idSites, $datesByMonth)
-    {
-        $yearMonths = array_keys($datesByMonth);
-        $yearMonths = array_unique($yearMonths);
-
-        $store = new InvalidatedReports();
-        $store->addInvalidatedSitesToReprocess($idSites);
-        $store->addSitesToPurgeForYearMonths($idSites, $yearMonths);
-    }
-
-    private static function getModel()
-    {
-        return new Model();
-    }
-
-} 
\ No newline at end of file
diff --git a/core/DataAccess/ArchivePurger.php b/core/DataAccess/ArchivePurger.php
deleted file mode 100644
index e7f185f475..0000000000
--- a/core/DataAccess/ArchivePurger.php
+++ /dev/null
@@ -1,139 +0,0 @@
-<?php
-/**
- * Piwik - free/libre analytics platform
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- *
- */
-namespace Piwik\DataAccess;
-
-use Exception;
-use Piwik\ArchiveProcessor\Rules;
-use Piwik\Config;
-use Piwik\Date;
-use Piwik\Db;
-use Piwik\Log;
-use Piwik\Piwik;
-
-/**
- *
- * This class purges two types of archives:
- *
- * (1) Deletes invalidated archives (from ArchiveInvalidator)
- *
- * (2) Deletes outdated archives (the temporary or errored archives)
- *
- *
- * @package Piwik\DataAccess
- */
-class ArchivePurger
-{
-    public static function purgeInvalidatedArchives()
-    {
-        $store = new InvalidatedReports();
-        $idSitesByYearMonth = $store->getSitesByYearMonthArchiveToPurge();
-        foreach ($idSitesByYearMonth as $yearMonth => $idSites) {
-            if(empty($idSites)) {
-                continue;
-            }
-
-            $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01');
-            $numericTable = ArchiveTableCreator::getNumericTable($date);
-
-            $archiveIds = self::getModel()->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
-
-            if (count($archiveIds) == 0) {
-                continue;
-            }
-            self::deleteArchiveIds($date, $archiveIds);
-
-            $store->markSiteIdsHaveBeenPurged($idSites, $yearMonth);
-        }
-    }
-
-    /**
-     * Removes the outdated archives for the given month.
-     * (meaning they are marked with a done flag of ArchiveWriter::DONE_OK_TEMPORARY or ArchiveWriter::DONE_ERROR)
-     *
-     * @param Date $dateStart Only the month will be used
-     */
-    public static function purgeOutdatedArchives(Date $dateStart)
-    {
-        $purgeArchivesOlderThan = Rules::shouldPurgeOutdatedArchives($dateStart);
-
-        if (!$purgeArchivesOlderThan) {
-            return;
-        }
-
-        $idArchivesToDelete = self::getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan);
-
-        if (!empty($idArchivesToDelete)) {
-            self::deleteArchiveIds($dateStart, $idArchivesToDelete);
-        }
-
-        self::deleteArchivesWithPeriodRange($dateStart);
-
-        Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]",
-                   $purgeArchivesOlderThan,
-                   $dateStart->toString("Y-m"),
-                   implode(',', $idArchivesToDelete));
-    }
-
-    protected static function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan)
-    {
-        $archiveTable = ArchiveTableCreator::getNumericTable($date);
-
-        $result = self::getModel()->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan);
-
-        $idArchivesToDelete = array();
-        if (!empty($result)) {
-            foreach ($result as $row) {
-                $idArchivesToDelete[] = $row['idarchive'];
-            }
-        }
-
-        return $idArchivesToDelete;
-    }
-
-    /**
-     * Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space.
-     *
-     * @param $date Date
-     */
-    protected static function deleteArchivesWithPeriodRange(Date $date)
-    {
-        $numericTable = ArchiveTableCreator::getNumericTable($date);
-        $blobTable    = ArchiveTableCreator::getBlobTable($date);
-        $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days'];
-        $pastDate    = Date::factory('today')->subDay($daysRangesValid)->getDateTime();
-
-        self::getModel()->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $pastDate);
-
-        Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]",
-            $pastDate, $numericTable);
-    }
-
-    /**
-     * Deletes by batches Archive IDs in the specified month,
-     *
-     * @param Date $date
-     * @param $idArchivesToDelete
-     */
-    protected static function deleteArchiveIds(Date $date, $idArchivesToDelete)
-    {
-        $batches      = array_chunk($idArchivesToDelete, 1000);
-        $numericTable = ArchiveTableCreator::getNumericTable($date);
-        $blobTable    = ArchiveTableCreator::getBlobTable($date);
-
-        foreach ($batches as $idsToDelete) {
-            self::getModel()->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
-        }
-    }
-
-    private static function getModel()
-    {
-        return new Model();
-    }
-
-}
diff --git a/core/DataAccess/InvalidatedReports.php b/core/DataAccess/InvalidatedReports.php
deleted file mode 100644
index 64f863e0ad..0000000000
--- a/core/DataAccess/InvalidatedReports.php
+++ /dev/null
@@ -1,168 +0,0 @@
-<?php
-/**
- * Piwik - free/libre analytics platform
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- *
- */
-
-namespace Piwik\DataAccess;
-
-use Piwik\Option;
-
-/**
- * Keeps track of which reports were invalidated via CoreAdminHome.invalidateArchivedReports API.
- *
- * This is used by:
- *
- * 1. core:archive command to know which websites should be reprocessed
- *
- * 2. scheduled task purgeInvalidatedArchives to know which websites/months should be purged
- *
- */
-class InvalidatedReports
-{
-    const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds';
-    const OPTION_INVALIDATED_DATES_SITES_TO_PURGE = 'InvalidatedOldReports_DatesWebsiteIds';
-
-    /**
-     * Mark the sites IDs and Dates as being invalidated, so we can purge them later on.
-     *
-     * @param array $idSites
-     * @param array $yearMonths
-     */
-    public function addSitesToPurgeForYearMonths(array $idSites, $yearMonths)
-    {
-        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
-
-        foreach($yearMonths as $yearMonthToPurge) {
-
-            if(isset($idSitesByYearMonth[$yearMonthToPurge])) {
-                $existingIdSitesToPurge = $idSitesByYearMonth[$yearMonthToPurge];
-                $idSites = array_merge($existingIdSitesToPurge, $idSites);
-                $idSites = array_unique($idSites);
-            }
-            $idSitesByYearMonth[$yearMonthToPurge] = $idSites;
-        }
-        $this->persistSitesByYearMonthToPurge($idSitesByYearMonth);
-    }
-
-    /**
-     * Returns the list of websites IDs for which invalidated archives can be purged.
-     */
-    public function getSitesByYearMonthArchiveToPurge()
-    {
-        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
-
-        // From this list we remove the websites that are not yet re-processed
-        // so we don't purge them before they were re-processed
-        $idSitesNotYetReprocessed = $this->getSitesToReprocess();
-
-        foreach($idSitesByYearMonth as $yearMonth => &$idSites) {
-            $idSites = array_diff($idSites, $idSitesNotYetReprocessed);
-        }
-        return $idSitesByYearMonth;
-    }
-
-    public function markSiteIdsHaveBeenPurged(array $idSites, $yearMonth)
-    {
-        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
-
-        if(!isset($idSitesByYearMonth[$yearMonth])) {
-            return;
-        }
-
-        $idSitesByYearMonth[$yearMonth] = array_diff($idSitesByYearMonth[$yearMonth], $idSites);
-        $this->persistSitesByYearMonthToPurge($idSitesByYearMonth);
-    }
-
-    /**
-     * Record those website IDs as having been invalidated
-     *
-     * @param $idSites
-     */
-    public function addInvalidatedSitesToReprocess(array $idSites)
-    {
-        $siteIdsToReprocess = $this->getSitesToReprocess();
-        $siteIdsToReprocess = array_merge($siteIdsToReprocess, $idSites);
-        $this->setSitesToReprocess($siteIdsToReprocess);
-    }
-
-
-    /**
-     * @param $idSite
-     */
-    public function storeSiteIsReprocessed($idSite)
-    {
-        $siteIdsToReprocess = $this->getSitesToReprocess();
-
-        if (count($siteIdsToReprocess)) {
-            $found = array_search($idSite, $siteIdsToReprocess);
-            if ($found !== false) {
-                unset($siteIdsToReprocess[$found]);
-                $this->setSitesToReprocess($siteIdsToReprocess);
-            }
-        }
-    }
-
-    /**
-     * Returns array of idSites to force re-process next time core:archive command runs
-     *
-     * @return array of id sites
-     */
-    public function getSitesToReprocess()
-    {
-        return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
-    }
-
-    /**
-     * @return array|false|mixed|string
-     */
-    private function getSitesByYearMonthToPurge()
-    {
-        return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE);
-    }
-
-    /**
-     * @param $websiteIdsInvalidated
-     */
-    private function setSitesToReprocess($websiteIdsInvalidated)
-    {
-        $websiteIdsInvalidated = array_unique($websiteIdsInvalidated);
-        $websiteIdsInvalidated = array_values($websiteIdsInvalidated);
-        Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated));
-    }
-
-    /**
-     * @param $optionName
-     * @return array|false|mixed|string
-     */
-    private function getArrayValueFromOptionName($optionName)
-    {
-        Option::clearCachedOption($optionName);
-        $array = Option::get($optionName);
-
-        if ($array
-            && ($array = unserialize($array))
-            && count($array)
-        ) {
-            return $array;
-        }
-        return array();
-    }
-
-    /**
-     * @param $idSitesByYearMonth
-     */
-    private function persistSitesByYearMonthToPurge($idSitesByYearMonth)
-    {
-        // remove dates for which there are no sites to purge
-        $idSitesByYearMonth = array_filter($idSitesByYearMonth);
-
-        Option::set(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE, serialize($idSitesByYearMonth));
-    }
-
-
-
-}
\ No newline at end of file
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index 239bce6a23..0980983fe3 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -246,6 +246,23 @@ class Model
     }
 
     /**
+     * Returns the site IDs for invalidated archives in an archive table.
+     *
+     * @param string $numericTable The numeric table to search through.
+     * @return int[]
+     */
+    public function getSitesWithInvalidatedArchive($numericTable)
+    {
+        $rows = Db::fetchAll("SELECT DISTINCT idsite FROM `$numericTable` WHERE name LIKE 'done%' AND value = " . ArchiveWriter::DONE_INVALIDATED);
+
+        $result = array();
+        foreach ($rows as $row) {
+            $result[] = $row['idsite'];
+        }
+        return $result;
+    }
+
+    /**
      * Returns the SQL condition used to find successfully completed archives that
      * this instance is querying for.
      */