Merge branch 'master' into 7276_update_command_progress

Conflicts:
	core/Console.php
	tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php

20 files changed, 601 insertions, 378 deletions

diff --git a/core/Archive.php b/core/Archive.php
index c7eb6e3106..2bce8c676d 100644
--- a/core/Archive.php
+++ b/core/Archive.php
@@ -10,7 +10,7 @@ namespace Piwik;
 
 use Piwik\Archive\Parameters;
 use Piwik\ArchiveProcessor\Rules;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\DataAccess\ArchiveSelector;
 use Piwik\Period\Factory as PeriodFactory;
 
@@ -498,7 +498,11 @@ class Archive
 
         $dataTable = self::getDataTableFromArchive($recordName, $idSite, $period, $date, $segment, $expanded, $idSubtable, $depth);
 
-        $dataTable->filter('ReplaceColumnNames');
+        $dataTable->queueFilter('ReplaceColumnNames');
+
+        if ($expanded) {
+            $dataTable->queueFilterSubtables('ReplaceColumnNames');
+        }
 
         if ($flat) {
             $dataTable->disableRecursiveFilters();
diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php
index 916d94991b..e9ce777cc4 100644
--- a/core/DataAccess/ArchiveInvalidator.php
+++ b/core/Archive/ArchiveInvalidator.php
@@ -7,11 +7,15 @@
  *
  */
 
-namespace Piwik\DataAccess;
+namespace Piwik\Archive;
 
+use Piwik\CronArchive\SitesToReprocessDistributedList;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\Model;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Option;
+use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
 use Piwik\Plugins\PrivacyManager\PrivacyManager;
 use Piwik\Period;
 use Piwik\Period\Week;
@@ -19,16 +23,30 @@ use Piwik\Plugins\SitesManager\Model as SitesManagerModel;
 use Piwik\Site;
 
 /**
- * Marks archives as Invalidated by setting the done flag to a special value (see Model->updateArchiveAsInvalidated)
+ * Service that can be used to invalidate archives or add archive references to a list so they will
+ * be invalidated later.
  *
- * Invalidated archives can still be selected and displayed in UI and API (until they are reprocessed by core:archive)
+ * Archives are put in an "invalidated" state by setting the done flag to `ArchiveWriter::DONE_INVALIDATED`.
+ * This class also adds the archive's associated site to the a distributed list and adding the archive's year month to another
+ * distributed list.
  *
- * The invalidated archives will be deleted by ArchivePurger
+ * CronArchive will reprocess the archive data for all sites in the first list, and a scheduled task
+ * will purge the old, invalidated data in archive tables identified by the second list.
  *
- * @package Piwik\DataAccess
+ * Until CronArchive, or browser triggered archiving, re-processes data for an invalidated archive, the invalidated
+ * archive data will still be displayed in the UI and API.
+ *
+ * ### Deferred Invalidation
+ *
+ * Invalidating archives means running queries on one or more archive tables. In some situations, like during
+ * tracking, this is not desired. In such cases, archive references can be added to a list via the
+ * rememberToInvalidateArchivedReportsLater method, which will add the reference to a distributed list
+ *
+ * Later, during Piwik's normal execution, the list will be read and every archive it references will
+ * be invalidated.
  */
-class ArchiveInvalidator {
-
+class ArchiveInvalidator
+{
     private $warningDates = array();
     private $processedDates = array();
     private $minimumDateWithLogs = false;
@@ -317,9 +335,11 @@ class ArchiveInvalidator {
         $yearMonths = array_keys($datesByMonth);
         $yearMonths = array_unique($yearMonths);
 
-        $store = new InvalidatedReports();
-        $store->addInvalidatedSitesToReprocess($idSites);
-        $store->addSitesToPurgeForYearMonths($idSites, $yearMonths);
+        $store = new SitesToReprocessDistributedList();
+        $store->add($idSites);
+
+        $archivesToPurge = new ArchivesToPurgeDistributedList();
+        $archivesToPurge->add($yearMonths);
     }
 
     private static function getModel()
diff --git a/core/Archive/ArchivePurger.php b/core/Archive/ArchivePurger.php
new file mode 100644
index 0000000000..e41ce375c5
--- /dev/null
+++ b/core/Archive/ArchivePurger.php
@@ -0,0 +1,227 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Archive;
+
+use Piwik\ArchiveProcessor\Rules;
+use Piwik\Config;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\Model;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Log;
+use Piwik\Piwik;
+
+/**
+ * Service that purges temporary, error-ed, invalid and custom range archives from archive tables.
+ *
+ * Temporary archives are purged if they were archived before a specific time. The time is dependent
+ * on whether browser triggered archiving is enabled or not.
+ *
+ * Error-ed archives are purged w/o constraint.
+ *
+ * Invalid archives are purged if a new, valid, archive exists w/ the same site, date, period combination.
+ * Archives are marked as invalid via Piwik\Archive\ArchiveInvalidator.
+ */
+class ArchivePurger
+{
+    /**
+     * @var Model
+     */
+    private $model;
+
+    /**
+     * Date threshold for purging custom range archives. Archives that are older than this date
+     * are purged unconditionally from the requested archive table.
+     *
+     * @var Date
+     */
+    private $purgeCustomRangesOlderThan;
+
+    /**
+     * Date to use for 'yesterday'. Exists so tests can override this value.
+     *
+     * @var Date
+     */
+    private $yesterday;
+
+    /**
+     * Date to use for 'today'. Exists so tests can override this value.
+     *
+     * @var $today
+     */
+    private $today;
+
+    /**
+     * Date to use for 'now'. Exists so tests can override this value.
+     *
+     * @var int
+     */
+    private $now;
+
+    public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null)
+    {
+        $this->model = $model ?: new Model();
+
+        $this->purgeCustomRangesOlderThan = $purgeCustomRangesOlderThan ?: self::getDefaultCustomRangeToPurgeAgeThreshold();
+
+        $this->yesterday = Date::factory('yesterday');
+        $this->today = Date::factory('today');
+        $this->now = time();
+    }
+
+    /**
+     * Purge all invalidate archives for whom there are newer, valid archives from the archive
+     * table that stores data for `$date`.
+     *
+     * @param Date $date The date identifying the archive table.
+     */
+    public function purgeInvalidatedArchivesFrom(Date $date)
+    {
+        $numericTable = ArchiveTableCreator::getNumericTable($date);
+
+        // we don't want to do an INNER JOIN on every row in a archive table that can potentially have tens to hundreds of thousands of rows,
+        // so we first look for sites w/ invalidated archives, and use this as a constraint in getInvalidatedArchiveIdsSafeToDelete() below.
+        // the constraint will hit an INDEX and speed up the inner join that happens in getInvalidatedArchiveIdsSafeToDelete().
+        $idSites = $this->model->getSitesWithInvalidatedArchive($numericTable);
+        if (empty($idSites)) {
+            return;
+        }
+
+        $archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
+        if (empty($archiveIds)) {
+            return;
+        }
+
+        $this->deleteArchiveIds($date, $archiveIds);
+    }
+
+    /**
+     * Removes the outdated archives for the given month.
+     * (meaning they are marked with a done flag of ArchiveWriter::DONE_OK_TEMPORARY or ArchiveWriter::DONE_ERROR)
+     *
+     * @param Date $dateStart Only the month will be used
+     */
+    public function purgeOutdatedArchives(Date $dateStart)
+    {
+        $purgeArchivesOlderThan = $this->getOldestTemporaryArchiveToKeepThreshold();
+
+        $idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan);
+        if (!empty($idArchivesToDelete)) {
+            $this->deleteArchiveIds($dateStart, $idArchivesToDelete);
+        }
+
+        Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]",
+                   $purgeArchivesOlderThan,
+                   $dateStart->toString("Y-m"),
+                   implode(',', $idArchivesToDelete));
+    }
+
+    protected function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan)
+    {
+        $archiveTable = ArchiveTableCreator::getNumericTable($date);
+
+        $result = $this->model->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan);
+
+        $idArchivesToDelete = array();
+        if (!empty($result)) {
+            foreach ($result as $row) {
+                $idArchivesToDelete[] = $row['idarchive'];
+            }
+        }
+
+        return $idArchivesToDelete;
+    }
+
+    /**
+     * Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space.
+     *
+     * @param $date Date
+     */
+    public function purgeArchivesWithPeriodRange(Date $date)
+    {
+        $numericTable = ArchiveTableCreator::getNumericTable($date);
+        $blobTable    = ArchiveTableCreator::getBlobTable($date);
+
+        $this->model->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan);
+
+        Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]",
+            $this->purgeCustomRangesOlderThan, $numericTable);
+    }
+
+    /**
+     * Deletes by batches Archive IDs in the specified month,
+     *
+     * @param Date $date
+     * @param $idArchivesToDelete
+     */
+    protected function deleteArchiveIds(Date $date, $idArchivesToDelete)
+    {
+        $batches      = array_chunk($idArchivesToDelete, 1000);
+        $numericTable = ArchiveTableCreator::getNumericTable($date);
+        $blobTable    = ArchiveTableCreator::getBlobTable($date);
+
+        foreach ($batches as $idsToDelete) {
+            $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
+        }
+    }
+
+    /**
+     * Returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged.
+     *
+     * @return int|bool  Outdated archives older than this timestamp should be purged
+     */
+    protected function getOldestTemporaryArchiveToKeepThreshold()
+    {
+        $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive();
+        if (Rules::isBrowserTriggerEnabled()) {
+            // If Browser Archiving is enabled, it is likely there are many more temporary archives
+            // We delete more often which is safe, since reports are re-processed on demand
+            return Date::factory($this->now - 2 * $temporaryArchivingTimeout)->getDateTime();
+        }
+
+        // If cron core:archive command is building the reports, we should keep all temporary reports from today
+        return $this->yesterday->getDateTime();
+    }
+
+    private static function getDefaultCustomRangeToPurgeAgeThreshold()
+    {
+        $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days'];
+        return Date::factory('today')->subDay($daysRangesValid)->getDateTime();
+    }
+
+    /**
+     * For tests.
+     *
+     * @param Date $yesterday
+     */
+    public function setYesterdayDate(Date $yesterday)
+    {
+        $this->yesterday = $yesterday;
+    }
+
+    /**
+     * For tests.
+     *
+     * @param Date $today
+     */
+    public function setTodayDate(Date $today)
+    {
+        $this->today = $today;
+    }
+
+    /**
+     * For tests.
+     *
+     * @param int $now
+     */
+    public function setNow($now)
+    {
+        $this->now = $now;
+    }
+}
\ No newline at end of file
diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php
index 06b38aa27d..4ad2fd1b25 100644
--- a/core/ArchiveProcessor/Rules.php
+++ b/core/ArchiveProcessor/Rules.php
@@ -10,7 +10,6 @@ namespace Piwik\ArchiveProcessor;
 
 use Exception;
 use Piwik\Config;
-use Piwik\Container\StaticContainer;
 use Piwik\DataAccess\ArchiveWriter;
 use Piwik\Date;
 use Piwik\Log;
@@ -113,37 +112,6 @@ class Rules
         return $doneFlags;
     }
 
-    /**
-     * Returns false if we should not purge data for this month,
-     * or returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged.
-     *
-     * Note: when calling this function it is assumed that the callee will purge the outdated archives afterwards.
-     *
-     * @param \Piwik\Date $date
-     * @return int|bool  Outdated archives older than this timestamp should be purged
-     */
-    public static function shouldPurgeOutdatedArchives(Date $date)
-    {
-        // we only delete archives if we are able to process them, otherwise, the browser might process reports
-        // when &segment= is specified (or custom date range) and would below, delete temporary archives that the
-        // browser is not able to process until next cron run (which could be more than 1 hour away)
-        if (! self::isRequestAuthorizedToArchive()){
-            Log::info("Purging temporary archives: skipped (no authorization)");
-            return false;
-        }
-
-        $temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
-
-        if (self::isBrowserTriggerEnabled()) {
-            // If Browser Archiving is enabled, it is likely there are many more temporary archives
-            // We delete more often which is safe, since reports are re-processed on demand
-            return Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime();
-        } 
-
-        // If cron core:archive command is building the reports, we should keep all temporary reports from today
-        return Date::factory('yesterday')->getDateTime();
-    }
-
     public static function getMinTimeProcessedForTemporaryArchive(
         Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site)
     {
@@ -309,5 +277,4 @@ class Rules
 
         return $possibleValues;
     }
-
-}
+}
\ No newline at end of file
diff --git a/core/Concurrency/DistributedList.php b/core/Concurrency/DistributedList.php
new file mode 100644
index 0000000000..eecfab6a55
--- /dev/null
+++ b/core/Concurrency/DistributedList.php
@@ -0,0 +1,138 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Concurrency;
+
+use Piwik\Option;
+
+/**
+ * Manages a simple distributed list stored in an Option. No locking occurs, so the list
+ * is not thread safe, and should only be used for use cases where atomicity is not
+ * important.
+ *
+ * The list of items is serialized and stored in an Option. Items are converted to string
+ * before being persisted, so it is not expected to unserialize objects.
+ */
+class DistributedList
+{
+    /**
+     * The name of the option to store the list in.
+     *
+     * @var string
+     */
+    private $optionName;
+
+    /**
+     * Constructor.
+     *
+     * @param string $optionName
+     */
+    public function __construct($optionName)
+    {
+        $this->optionName = $optionName;
+    }
+
+    /**
+     * Queries the option table and returns all items in this list.
+     *
+     * @return array
+     */
+    public function getAll()
+    {
+        Option::clearCachedOption($this->optionName);
+        $array = Option::get($this->optionName);
+
+        if ($array
+            && ($array = unserialize($array))
+            && count($array)
+        ) {
+            return $array;
+        }
+        return array();
+    }
+
+    /**
+     * Sets the contents of the list in the option table.
+     *
+     * @param string[] $items
+     */
+    public function setAll($items)
+    {
+        foreach ($items as &$item) {
+            $item = (string)$item;
+        }
+
+        Option::set($this->optionName, serialize($items));
+    }
+
+    /**
+     * Adds one or more items to the list in the option table.
+     *
+     * @param string|array $item
+     */
+    public function add($item)
+    {
+        $allItems = $this->getAll();
+        if (is_array($item)) {
+            $allItems = array_merge($allItems, $item);
+        } else {
+            $allItems[] = $item;
+        }
+
+        $this->setAll($allItems);
+    }
+
+    /**
+     * Removes one or more items by value from the list in the option table.
+     *
+     * Does not preserve array keys.
+     *
+     * @param string|array $items
+     */
+    public function remove($items)
+    {
+        if (!is_array($items)) {
+            $items = array($items);
+        }
+
+        $allItems = $this->getAll();
+
+        foreach ($items as $item) {
+            $existingIndex = array_search($item, $allItems);
+            if ($existingIndex === false) {
+                return;
+            }
+
+            unset($allItems[$existingIndex]);
+        }
+
+        $this->setAll(array_values($allItems));
+    }
+
+    /**
+     * Removes one or more items by index from the list in the option table.
+     *
+     * Does not preserve array keys.
+     *
+     * @param int[]|int $indices
+     */
+    public function removeByIndex($indices)
+    {
+        if (!is_array($indices)) {
+            $indices = array($indices);
+        }
+
+        $indices = array_unique($indices);
+
+        $allItems = $this->getAll();
+        foreach ($indices as $index) {
+            unset($allItems[$index]);
+        }
+
+        $this->setAll(array_values($allItems));
+    }
+}
\ No newline at end of file
diff --git a/core/Config.php b/core/Config.php
index e28f581ea8..f0b5c3ed26 100644
--- a/core/Config.php
+++ b/core/Config.php
@@ -10,6 +10,7 @@
 namespace Piwik;
 
 use Exception;
+use Piwik\Config\ConfigNotFoundException;
 use Piwik\Ini\IniReader;
 use Piwik\Ini\IniReadingException;
 use Piwik\Ini\IniWriter;
@@ -370,7 +371,7 @@ class Config extends Singleton
     public function checkLocalConfigFound()
     {
         if (!$this->existsLocalConfig()) {
-            throw new Exception(Piwik::translate('General_ExceptionConfigurationFileNotFound', array($this->pathLocal)));
+            throw new ConfigNotFoundException(Piwik::translate('General_ExceptionConfigurationFileNotFound', array($this->pathLocal)));
         }
     }
 
diff --git a/core/Config/ConfigNotFoundException.php b/core/Config/ConfigNotFoundException.php
new file mode 100644
index 0000000000..5860367ba7
--- /dev/null
+++ b/core/Config/ConfigNotFoundException.php
@@ -0,0 +1,16 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Config;
+
+/**
+ * Exception thrown when the config file doesn't exist.
+ */
+class ConfigNotFoundException extends \Exception
+{
+}
diff --git a/core/Console.php b/core/Console.php
index 02e5f86871..cfed85a994 100644
--- a/core/Console.php
+++ b/core/Console.php
@@ -8,10 +8,12 @@
  */
 namespace Piwik;
 
+use Piwik\Config\ConfigNotFoundException;
 use Piwik\Container\StaticContainer;
 use Piwik\Plugin\Manager as PluginManager;
 use Symfony\Bridge\Monolog\Handler\ConsoleHandler;
 use Symfony\Component\Console\Application;
+use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -43,10 +45,9 @@ class Console extends Application
 
         try {
             self::initPlugins();
-        } catch(\Exception $e) {
+        } catch (ConfigNotFoundException $e) {
             // Piwik not installed yet, no config file?
-
-            Log::debug("Could not initialize plugins: " . $e->getMessage() . "\n" . $e->getTraceAsString());
+            Log::warning($e->getMessage());
         }
 
         $commands = $this->getAvailableCommands();
@@ -65,10 +66,16 @@ class Console extends Application
     {
         if (!class_exists($command)) {
             Log::warning(sprintf('Cannot add command %s, class does not exist', $command));
-        } elseif (!is_subclass_of($command, 'Piwik\Plugin\ConsoleCommand')) {
+        } else if (!is_subclass_of($command, 'Piwik\Plugin\ConsoleCommand')) {
             Log::warning(sprintf('Cannot add command %s, class does not extend Piwik\Plugin\ConsoleCommand', $command));
         } else {
-            $this->add(new $command);
+            /** @var Command $commandInstance */
+            $commandInstance = new $command;
+
+            // do not add the command if it already exists; this way we can add the command ourselves in tests
+            if (!$this->has($commandInstance->getName())) {
+                $this->add($commandInstance);
+            }
         }
     }
 
diff --git a/core/CronArchive.php b/core/CronArchive.php
index 29c897c8bf..0ff2ee8242 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -12,11 +12,11 @@ use Exception;
 use Piwik\ArchiveProcessor\Rules;
 use Piwik\CronArchive\FixedSiteIds;
 use Piwik\CronArchive\SharedSiteIds;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Metrics\Formatter;
 use Piwik\Period\Factory as PeriodFactory;
-use Piwik\DataAccess\InvalidatedReports;
+use Piwik\CronArchive\SitesToReprocessDistributedList;
 use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI;
 use Piwik\Plugins\SitesManager\API as APISitesManager;
 use Piwik\Plugins\UsersManager\API as APIUsersManager;
@@ -542,8 +542,8 @@ class CronArchive
         if(!$success) {
             // cancel marking the site as reprocessed
             if($websiteInvalidatedShouldReprocess) {
-                $store = new InvalidatedReports();
-                $store->addInvalidatedSitesToReprocess(array($idSite));
+                $store = new SitesToReprocessDistributedList();
+                $store->add($idSite);
             }
         }
 
@@ -657,8 +657,8 @@ class CronArchive
         // does not archive the same idSite
         $websiteInvalidatedShouldReprocess = $this->isOldReportInvalidatedForWebsite($idSite);
         if ($websiteInvalidatedShouldReprocess) {
-            $store = new InvalidatedReports();
-            $store->storeSiteIsReprocessed($idSite);
+            $store = new SitesToReprocessDistributedList();
+            $store->remove($idSite);
         }
 
         // when some data was purged from this website
@@ -685,8 +685,8 @@ class CronArchive
 
             // cancel marking the site as reprocessed
             if($websiteInvalidatedShouldReprocess) {
-                $store = new InvalidatedReports();
-                $store->addInvalidatedSitesToReprocess(array($idSite));
+                $store = new SitesToReprocessDistributedList();
+                $store->add($idSite);
             }
 
             $this->logError("Empty or invalid response '$content' for website id $idSite, " . $timerWebsite->__toString() . ", skipping");
@@ -1083,8 +1083,8 @@ class CronArchive
 
     private function updateIdSitesInvalidatedOldReports()
     {
-        $store = new InvalidatedReports();
-        $this->idSitesInvalidatedOldReports = $store->getSitesToReprocess();
+        $store = new SitesToReprocessDistributedList();
+        $this->idSitesInvalidatedOldReports = $store->getAll();
     }
 
     /**
diff --git a/core/CronArchive/SitesToReprocessDistributedList.php b/core/CronArchive/SitesToReprocessDistributedList.php
new file mode 100644
index 0000000000..44b1eedca6
--- /dev/null
+++ b/core/CronArchive/SitesToReprocessDistributedList.php
@@ -0,0 +1,40 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\CronArchive;
+
+use Piwik\Concurrency\DistributedList;
+
+/**
+ * Distributed list that stores the list of IDs of sites whose archives should be reprocessed.
+ *
+ * CronArchive will read this list of sites when archiving is being run, and make sure the sites
+ * are re-archived.
+ *
+ * Any class/API method/command/etc. is allowed to add site IDs to this list.
+ */
+class SitesToReprocessDistributedList extends DistributedList
+{
+    const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds';
+
+    public function __construct()
+    {
+        parent::__construct(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function setAll($items)
+    {
+        $items = array_unique($items);
+        $items = array_values($items);
+
+        parent::setAll($items);
+    }
+}
\ No newline at end of file
diff --git a/core/DataAccess/ArchivePurger.php b/core/DataAccess/ArchivePurger.php
deleted file mode 100644
index e7f185f475..0000000000
--- a/core/DataAccess/ArchivePurger.php
+++ /dev/null
@@ -1,139 +0,0 @@
-<?php
-/**
- * Piwik - free/libre analytics platform
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- *
- */
-namespace Piwik\DataAccess;
-
-use Exception;
-use Piwik\ArchiveProcessor\Rules;
-use Piwik\Config;
-use Piwik\Date;
-use Piwik\Db;
-use Piwik\Log;
-use Piwik\Piwik;
-
-/**
- *
- * This class purges two types of archives:
- *
- * (1) Deletes invalidated archives (from ArchiveInvalidator)
- *
- * (2) Deletes outdated archives (the temporary or errored archives)
- *
- *
- * @package Piwik\DataAccess
- */
-class ArchivePurger
-{
-    public static function purgeInvalidatedArchives()
-    {
-        $store = new InvalidatedReports();
-        $idSitesByYearMonth = $store->getSitesByYearMonthArchiveToPurge();
-        foreach ($idSitesByYearMonth as $yearMonth => $idSites) {
-            if(empty($idSites)) {
-                continue;
-            }
-
-            $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01');
-            $numericTable = ArchiveTableCreator::getNumericTable($date);
-
-            $archiveIds = self::getModel()->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
-
-            if (count($archiveIds) == 0) {
-                continue;
-            }
-            self::deleteArchiveIds($date, $archiveIds);
-
-            $store->markSiteIdsHaveBeenPurged($idSites, $yearMonth);
-        }
-    }
-
-    /**
-     * Removes the outdated archives for the given month.
-     * (meaning they are marked with a done flag of ArchiveWriter::DONE_OK_TEMPORARY or ArchiveWriter::DONE_ERROR)
-     *
-     * @param Date $dateStart Only the month will be used
-     */
-    public static function purgeOutdatedArchives(Date $dateStart)
-    {
-        $purgeArchivesOlderThan = Rules::shouldPurgeOutdatedArchives($dateStart);
-
-        if (!$purgeArchivesOlderThan) {
-            return;
-        }
-
-        $idArchivesToDelete = self::getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan);
-
-        if (!empty($idArchivesToDelete)) {
-            self::deleteArchiveIds($dateStart, $idArchivesToDelete);
-        }
-
-        self::deleteArchivesWithPeriodRange($dateStart);
-
-        Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]",
-                   $purgeArchivesOlderThan,
-                   $dateStart->toString("Y-m"),
-                   implode(',', $idArchivesToDelete));
-    }
-
-    protected static function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan)
-    {
-        $archiveTable = ArchiveTableCreator::getNumericTable($date);
-
-        $result = self::getModel()->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan);
-
-        $idArchivesToDelete = array();
-        if (!empty($result)) {
-            foreach ($result as $row) {
-                $idArchivesToDelete[] = $row['idarchive'];
-            }
-        }
-
-        return $idArchivesToDelete;
-    }
-
-    /**
-     * Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space.
-     *
-     * @param $date Date
-     */
-    protected static function deleteArchivesWithPeriodRange(Date $date)
-    {
-        $numericTable = ArchiveTableCreator::getNumericTable($date);
-        $blobTable    = ArchiveTableCreator::getBlobTable($date);
-        $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days'];
-        $pastDate    = Date::factory('today')->subDay($daysRangesValid)->getDateTime();
-
-        self::getModel()->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $pastDate);
-
-        Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]",
-            $pastDate, $numericTable);
-    }
-
-    /**
-     * Deletes by batches Archive IDs in the specified month,
-     *
-     * @param Date $date
-     * @param $idArchivesToDelete
-     */
-    protected static function deleteArchiveIds(Date $date, $idArchivesToDelete)
-    {
-        $batches      = array_chunk($idArchivesToDelete, 1000);
-        $numericTable = ArchiveTableCreator::getNumericTable($date);
-        $blobTable    = ArchiveTableCreator::getBlobTable($date);
-
-        foreach ($batches as $idsToDelete) {
-            self::getModel()->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
-        }
-    }
-
-    private static function getModel()
-    {
-        return new Model();
-    }
-
-}
diff --git a/core/DataAccess/InvalidatedReports.php b/core/DataAccess/InvalidatedReports.php
deleted file mode 100644
index 64f863e0ad..0000000000
--- a/core/DataAccess/InvalidatedReports.php
+++ /dev/null
@@ -1,168 +0,0 @@
-<?php
-/**
- * Piwik - free/libre analytics platform
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- *
- */
-
-namespace Piwik\DataAccess;
-
-use Piwik\Option;
-
-/**
- * Keeps track of which reports were invalidated via CoreAdminHome.invalidateArchivedReports API.
- *
- * This is used by:
- *
- * 1. core:archive command to know which websites should be reprocessed
- *
- * 2. scheduled task purgeInvalidatedArchives to know which websites/months should be purged
- *
- */
-class InvalidatedReports
-{
-    const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds';
-    const OPTION_INVALIDATED_DATES_SITES_TO_PURGE = 'InvalidatedOldReports_DatesWebsiteIds';
-
-    /**
-     * Mark the sites IDs and Dates as being invalidated, so we can purge them later on.
-     *
-     * @param array $idSites
-     * @param array $yearMonths
-     */
-    public function addSitesToPurgeForYearMonths(array $idSites, $yearMonths)
-    {
-        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
-
-        foreach($yearMonths as $yearMonthToPurge) {
-
-            if(isset($idSitesByYearMonth[$yearMonthToPurge])) {
-                $existingIdSitesToPurge = $idSitesByYearMonth[$yearMonthToPurge];
-                $idSites = array_merge($existingIdSitesToPurge, $idSites);
-                $idSites = array_unique($idSites);
-            }
-            $idSitesByYearMonth[$yearMonthToPurge] = $idSites;
-        }
-        $this->persistSitesByYearMonthToPurge($idSitesByYearMonth);
-    }
-
-    /**
-     * Returns the list of websites IDs for which invalidated archives can be purged.
-     */
-    public function getSitesByYearMonthArchiveToPurge()
-    {
-        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
-
-        // From this list we remove the websites that are not yet re-processed
-        // so we don't purge them before they were re-processed
-        $idSitesNotYetReprocessed = $this->getSitesToReprocess();
-
-        foreach($idSitesByYearMonth as $yearMonth => &$idSites) {
-            $idSites = array_diff($idSites, $idSitesNotYetReprocessed);
-        }
-        return $idSitesByYearMonth;
-    }
-
-    public function markSiteIdsHaveBeenPurged(array $idSites, $yearMonth)
-    {
-        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
-
-        if(!isset($idSitesByYearMonth[$yearMonth])) {
-            return;
-        }
-
-        $idSitesByYearMonth[$yearMonth] = array_diff($idSitesByYearMonth[$yearMonth], $idSites);
-        $this->persistSitesByYearMonthToPurge($idSitesByYearMonth);
-    }
-
-    /**
-     * Record those website IDs as having been invalidated
-     *
-     * @param $idSites
-     */
-    public function addInvalidatedSitesToReprocess(array $idSites)
-    {
-        $siteIdsToReprocess = $this->getSitesToReprocess();
-        $siteIdsToReprocess = array_merge($siteIdsToReprocess, $idSites);
-        $this->setSitesToReprocess($siteIdsToReprocess);
-    }
-
-
-    /**
-     * @param $idSite
-     */
-    public function storeSiteIsReprocessed($idSite)
-    {
-        $siteIdsToReprocess = $this->getSitesToReprocess();
-
-        if (count($siteIdsToReprocess)) {
-            $found = array_search($idSite, $siteIdsToReprocess);
-            if ($found !== false) {
-                unset($siteIdsToReprocess[$found]);
-                $this->setSitesToReprocess($siteIdsToReprocess);
-            }
-        }
-    }
-
-    /**
-     * Returns array of idSites to force re-process next time core:archive command runs
-     *
-     * @return array of id sites
-     */
-    public function getSitesToReprocess()
-    {
-        return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
-    }
-
-    /**
-     * @return array|false|mixed|string
-     */
-    private function getSitesByYearMonthToPurge()
-    {
-        return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE);
-    }
-
-    /**
-     * @param $websiteIdsInvalidated
-     */
-    private function setSitesToReprocess($websiteIdsInvalidated)
-    {
-        $websiteIdsInvalidated = array_unique($websiteIdsInvalidated);
-        $websiteIdsInvalidated = array_values($websiteIdsInvalidated);
-        Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated));
-    }
-
-    /**
-     * @param $optionName
-     * @return array|false|mixed|string
-     */
-    private function getArrayValueFromOptionName($optionName)
-    {
-        Option::clearCachedOption($optionName);
-        $array = Option::get($optionName);
-
-        if ($array
-            && ($array = unserialize($array))
-            && count($array)
-        ) {
-            return $array;
-        }
-        return array();
-    }
-
-    /**
-     * @param $idSitesByYearMonth
-     */
-    private function persistSitesByYearMonthToPurge($idSitesByYearMonth)
-    {
-        // remove dates for which there are no sites to purge
-        $idSitesByYearMonth = array_filter($idSitesByYearMonth);
-
-        Option::set(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE, serialize($idSitesByYearMonth));
-    }
-
-
-
-}
\ No newline at end of file
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index 239bce6a23..0980983fe3 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -246,6 +246,23 @@ class Model
     }
 
     /**
+     * Returns the site IDs for invalidated archives in an archive table.
+     *
+     * @param string $numericTable The numeric table to search through.
+     * @return int[]
+     */
+    public function getSitesWithInvalidatedArchive($numericTable)
+    {
+        $rows = Db::fetchAll("SELECT DISTINCT idsite FROM `$numericTable` WHERE name LIKE 'done%' AND value = " . ArchiveWriter::DONE_INVALIDATED);
+
+        $result = array();
+        foreach ($rows as $row) {
+            $result[] = $row['idsite'];
+        }
+        return $result;
+    }
+
+    /**
      * Returns the SQL condition used to find successfully completed archives that
      * this instance is querying for.
      */
diff --git a/core/DataTable.php b/core/DataTable.php
index 0b0d9845f0..48f4c1426b 100644
--- a/core/DataTable.php
+++ b/core/DataTable.php
@@ -489,6 +489,27 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess
     }
 
     /**
+     * Adds a filter and a list of parameters to the list of queued filters of all subtables. These filters will be
+     * executed when {@link applyQueuedFilters()} is called.
+     *
+     * Filters that prettify the column values or don't need the full set of rows should be queued. This
+     * way they will be run after the table is truncated which will result in better performance.
+     *
+     * @param string|Closure $className The class name of the filter, eg. `'Limit'`.
+     * @param array $parameters The parameters to give to the filter, eg. `array($offset, $limit)` for the Limit filter.
+     */
+    public function queueFilterSubtables($className, $parameters = array())
+    {
+        foreach ($this->getRows() as $row) {
+            $subtable = $row->getSubtable();
+            if ($subtable) {
+                $subtable->queueFilter($className, $parameters);
+                $subtable->queueFilterSubtables($className, $parameters);
+            }
+        }
+    }
+
+    /**
      * Adds a filter and a list of parameters to the list of queued filters. These filters will be
      * executed when {@link applyQueuedFilters()} is called.
      *
@@ -1733,4 +1754,4 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess
     {
         $this->deleteRow($offset);
     }
-}
\ No newline at end of file
+}
diff --git a/core/DataTable/Filter/Sort.php b/core/DataTable/Filter/Sort.php
index 632da35dc8..3da11b4119 100644
--- a/core/DataTable/Filter/Sort.php
+++ b/core/DataTable/Filter/Sort.php
@@ -143,9 +143,9 @@ Sort extends BaseFilter
             );
     }
 
-    protected function getColumnValue(Row $table )
+    protected function getColumnValue(Row $row)
     {
-        $value = $table->getColumn($this->columnToSort);
+        $value = $row->getColumn($this->columnToSort);
 
         if ($value === false
             || is_array($value)
diff --git a/core/DataTable/Map.php b/core/DataTable/Map.php
index 8787b06404..ccf201c5be 100644
--- a/core/DataTable/Map.php
+++ b/core/DataTable/Map.php
@@ -123,6 +123,19 @@ class Map implements DataTableInterface
     }
 
     /**
+     * Apply a queued filter to all subtables contained by this instance.
+     *
+     * @param string|Closure $className Name of filter class or a Closure.
+     * @param array $parameters Parameters to pass to the filter.
+     */
+    public function queueFilterSubtables($className, $parameters = array())
+    {
+        foreach ($this->getDataTables() as $table) {
+            $table->queueFilterSubtables($className, $parameters);
+        }
+    }
+
+    /**
      * Returns the array of DataTables contained by this class.
      *
      * @return DataTable[]|Map[]
@@ -174,6 +187,20 @@ class Map implements DataTableInterface
         $this->array[$label] = $table;
     }
 
+    public function getRowFromIdSubDataTable($idSubtable)
+    {
+        $dataTables = $this->getDataTables();
+
+        // find first datatable containing data
+        foreach ($dataTables as $subTable) {
+            $subTableRow = $subTable->getRowFromIdSubDataTable($idSubtable);
+
+            if (!empty($subTableRow)) {
+                return $subTableRow;
+            }
+        }
+    }
+
     /**
      * Returns a string output of this DataTable\Map (applying the default renderer to every {@link DataTable}
      * of this DataTable\Map).
diff --git a/core/Http.php b/core/Http.php
index 08bcbcef15..ffd0bf4383 100644
--- a/core/Http.php
+++ b/core/Http.php
@@ -64,6 +64,9 @@ class Http
      *                              Doesn't work w/ `fopen` transport method.
      * @param bool $getExtendedInfo If true returns the status code, headers & response, if false just the response.
      * @param string $httpMethod The HTTP method to use. Defaults to `'GET'`.
+     * @param string $httpUsername HTTP Auth username
+     * @param string $httpPassword HTTP Auth password
+     *
      * @throws Exception if the response cannot be saved to `$destinationPath`, if the HTTP response cannot be sent,
      *                   if there are more than 5 redirects or if the request times out.
      * @return bool|string If `$destinationPath` is not specified the HTTP response is returned on success. `false`
@@ -78,7 +81,17 @@ class Http
      *                     `false` is still returned on failure.
      * @api
      */
-    public static function sendHttpRequest($aUrl, $timeout, $userAgent = null, $destinationPath = null, $followDepth = 0, $acceptLanguage = false, $byteRange = false, $getExtendedInfo = false, $httpMethod = 'GET')
+    public static function sendHttpRequest($aUrl,
+                                           $timeout,
+                                           $userAgent = null,
+                                           $destinationPath = null,
+                                           $followDepth = 0,
+                                           $acceptLanguage = false,
+                                           $byteRange = false,
+                                           $getExtendedInfo = false,
+                                           $httpMethod = 'GET',
+                                           $httpUsername = null,
+                                           $httpPassword = null)
     {
         // create output file
         $file = null;
@@ -91,7 +104,7 @@ class Http
         }
 
         $acceptLanguage = $acceptLanguage ? 'Accept-Language: ' . $acceptLanguage : '';
-        return self::sendHttpRequestBy(self::getTransportMethod(), $aUrl, $timeout, $userAgent, $destinationPath, $file, $followDepth, $acceptLanguage, $acceptInvalidSslCertificate = false, $byteRange, $getExtendedInfo, $httpMethod);
+        return self::sendHttpRequestBy(self::getTransportMethod(), $aUrl, $timeout, $userAgent, $destinationPath, $file, $followDepth, $acceptLanguage, $acceptInvalidSslCertificate = false, $byteRange, $getExtendedInfo, $httpMethod, $httpUsername, $httpPassword);
     }
 
     /**
@@ -110,6 +123,8 @@ class Http
      *                                                  Doesn't work w/ fopen method.
      * @param bool $getExtendedInfo True to return status code, headers & response, false if just response.
      * @param string $httpMethod The HTTP method to use. Defaults to `'GET'`.
+     * @param string $httpUsername HTTP Auth username
+     * @param string $httpPassword HTTP Auth password
      *
      * @throws Exception
      * @return bool  true (or string/array) on success; false on HTTP response error code (1xx or 4xx)
@@ -126,7 +141,9 @@ class Http
         $acceptInvalidSslCertificate = false,
         $byteRange = false,
         $getExtendedInfo = false,
-        $httpMethod = 'GET'
+        $httpMethod = 'GET',
+        $httpUsername = null,
+        $httpPassword = null
     )
     {
         if ($followDepth > 5) {
@@ -168,6 +185,11 @@ class Http
         $status  = null;
         $headers = array();
 
+        $httpAuthIsUsed = !empty($httpUsername) || !empty($httpPassword);
+        if($httpAuthIsUsed && $method != 'curl') {
+            throw new Exception("Specifying HTTP Username and HTTP password is only supported for CURL for now.");
+        }
+
         if ($method == 'socket') {
             if (!self::isSocketEnabled()) {
                 // can be triggered in tests
@@ -315,7 +337,9 @@ class Http
                         $acceptInvalidSslCertificate = false,
                         $byteRange,
                         $getExtendedInfo,
-                        $httpMethod
+                        $httpMethod,
+                        $httpUsername,
+                        $httpPassword
                     );
                 }
 
@@ -462,6 +486,12 @@ class Http
                 @curl_setopt($ch, CURLOPT_NOBODY, true);
             }
 
+            if(!empty($httpUsername) && !empty($httpPassword)) {
+                $curl_options += array(
+                    CURLOPT_USERPWD => $httpUsername . ':' . $httpPassword,
+                );
+            }
+
             @curl_setopt_array($ch, $curl_options);
             self::configCurlCertificate($ch);
 
diff --git a/core/Tracker/Model.php b/core/Tracker/Model.php
index e40f845e00..bb4fc1075a 100644
--- a/core/Tracker/Model.php
+++ b/core/Tracker/Model.php
@@ -391,6 +391,21 @@ class Model
         return $visitRow;
     }
 
+    /**
+     * Returns true if the site doesn't have log data.
+     *
+     * @param int $siteId
+     * @return bool
+     */
+    public function isSiteEmpty($siteId)
+    {
+        $sql = sprintf('SELECT idsite FROM %s WHERE idsite = ? limit 1', Common::prefixTable('log_visit'));
+
+        $result = \Piwik\Db::fetchOne($sql, array($siteId));
+
+        return $result == null;
+    }
+
     private function visitFieldsToQuery($valuesToUpdate)
     {
         $updateParts = array();
diff --git a/core/Tracker/Visit.php b/core/Tracker/Visit.php
index e5dbc69ef6..f672cd96da 100644
--- a/core/Tracker/Visit.php
+++ b/core/Tracker/Visit.php
@@ -11,7 +11,7 @@ namespace Piwik\Tracker;
 
 use Piwik\Common;
 use Piwik\Config;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Date;
 use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Network\IPUtils;
diff --git a/core/Version.php b/core/Version.php
index 0d5a9e14b9..3474cbf56b 100644
--- a/core/Version.php
+++ b/core/Version.php
@@ -20,7 +20,7 @@ final class Version
      * The current Piwik version.
      * @var string
      */
-    const VERSION = '2.12.0-b1';
+    const VERSION = '2.12.0-b3';
 
     public function isStableVersion($version)
     {