Merge pull request #7377 from piwik/7181_isolated_archive_purging

refactor archive purging for clarity and resilience.

31 files changed, 1708 insertions, 386 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 82a9d15f10..321875df34 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -25,6 +25,9 @@ This is a changelog for Piwik platform developers. All changes for our HTTP API'
   * `isIpInRange()`
   * `getHostByAddr()`
 
+### New commands
+* There is now a command `core:purge-old-archive-data` that can be used to manually purge temporary, error-ed and invalidated archives from one or more archive tables.
+
 ## Piwik 2.11.0
 
 ### Breaking Changes
diff --git a/core/Archive.php b/core/Archive.php
index 0499756410..2bce8c676d 100644
--- a/core/Archive.php
+++ b/core/Archive.php
@@ -10,7 +10,7 @@ namespace Piwik;
 
 use Piwik\Archive\Parameters;
 use Piwik\ArchiveProcessor\Rules;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\DataAccess\ArchiveSelector;
 use Piwik\Period\Factory as PeriodFactory;
 
diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/Archive/ArchiveInvalidator.php
index 916d94991b..e9ce777cc4 100644
--- a/core/DataAccess/ArchiveInvalidator.php
+++ b/core/Archive/ArchiveInvalidator.php
@@ -7,11 +7,15 @@
  *
  */
 
-namespace Piwik\DataAccess;
+namespace Piwik\Archive;
 
+use Piwik\CronArchive\SitesToReprocessDistributedList;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\Model;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Option;
+use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
 use Piwik\Plugins\PrivacyManager\PrivacyManager;
 use Piwik\Period;
 use Piwik\Period\Week;
@@ -19,16 +23,30 @@ use Piwik\Plugins\SitesManager\Model as SitesManagerModel;
 use Piwik\Site;
 
 /**
- * Marks archives as Invalidated by setting the done flag to a special value (see Model->updateArchiveAsInvalidated)
+ * Service that can be used to invalidate archives or add archive references to a list so they will
+ * be invalidated later.
  *
- * Invalidated archives can still be selected and displayed in UI and API (until they are reprocessed by core:archive)
+ * Archives are put in an "invalidated" state by setting the done flag to `ArchiveWriter::DONE_INVALIDATED`.
+ * This class also adds the archive's associated site to the a distributed list and adding the archive's year month to another
+ * distributed list.
  *
- * The invalidated archives will be deleted by ArchivePurger
+ * CronArchive will reprocess the archive data for all sites in the first list, and a scheduled task
+ * will purge the old, invalidated data in archive tables identified by the second list.
  *
- * @package Piwik\DataAccess
+ * Until CronArchive, or browser triggered archiving, re-processes data for an invalidated archive, the invalidated
+ * archive data will still be displayed in the UI and API.
+ *
+ * ### Deferred Invalidation
+ *
+ * Invalidating archives means running queries on one or more archive tables. In some situations, like during
+ * tracking, this is not desired. In such cases, archive references can be added to a list via the
+ * rememberToInvalidateArchivedReportsLater method, which will add the reference to a distributed list
+ *
+ * Later, during Piwik's normal execution, the list will be read and every archive it references will
+ * be invalidated.
  */
-class ArchiveInvalidator {
-
+class ArchiveInvalidator
+{
     private $warningDates = array();
     private $processedDates = array();
     private $minimumDateWithLogs = false;
@@ -317,9 +335,11 @@ class ArchiveInvalidator {
         $yearMonths = array_keys($datesByMonth);
         $yearMonths = array_unique($yearMonths);
 
-        $store = new InvalidatedReports();
-        $store->addInvalidatedSitesToReprocess($idSites);
-        $store->addSitesToPurgeForYearMonths($idSites, $yearMonths);
+        $store = new SitesToReprocessDistributedList();
+        $store->add($idSites);
+
+        $archivesToPurge = new ArchivesToPurgeDistributedList();
+        $archivesToPurge->add($yearMonths);
     }
 
     private static function getModel()
diff --git a/core/Archive/ArchivePurger.php b/core/Archive/ArchivePurger.php
new file mode 100644
index 0000000000..e41ce375c5
--- /dev/null
+++ b/core/Archive/ArchivePurger.php
@@ -0,0 +1,227 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Archive;
+
+use Piwik\ArchiveProcessor\Rules;
+use Piwik\Config;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\Model;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Log;
+use Piwik\Piwik;
+
+/**
+ * Service that purges temporary, error-ed, invalid and custom range archives from archive tables.
+ *
+ * Temporary archives are purged if they were archived before a specific time. The time is dependent
+ * on whether browser triggered archiving is enabled or not.
+ *
+ * Error-ed archives are purged w/o constraint.
+ *
+ * Invalid archives are purged if a new, valid, archive exists w/ the same site, date, period combination.
+ * Archives are marked as invalid via Piwik\Archive\ArchiveInvalidator.
+ */
+class ArchivePurger
+{
+    /**
+     * @var Model
+     */
+    private $model;
+
+    /**
+     * Date threshold for purging custom range archives. Archives that are older than this date
+     * are purged unconditionally from the requested archive table.
+     *
+     * @var Date
+     */
+    private $purgeCustomRangesOlderThan;
+
+    /**
+     * Date to use for 'yesterday'. Exists so tests can override this value.
+     *
+     * @var Date
+     */
+    private $yesterday;
+
+    /**
+     * Date to use for 'today'. Exists so tests can override this value.
+     *
+     * @var $today
+     */
+    private $today;
+
+    /**
+     * Date to use for 'now'. Exists so tests can override this value.
+     *
+     * @var int
+     */
+    private $now;
+
+    public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null)
+    {
+        $this->model = $model ?: new Model();
+
+        $this->purgeCustomRangesOlderThan = $purgeCustomRangesOlderThan ?: self::getDefaultCustomRangeToPurgeAgeThreshold();
+
+        $this->yesterday = Date::factory('yesterday');
+        $this->today = Date::factory('today');
+        $this->now = time();
+    }
+
+    /**
+     * Purge all invalidate archives for whom there are newer, valid archives from the archive
+     * table that stores data for `$date`.
+     *
+     * @param Date $date The date identifying the archive table.
+     */
+    public function purgeInvalidatedArchivesFrom(Date $date)
+    {
+        $numericTable = ArchiveTableCreator::getNumericTable($date);
+
+        // we don't want to do an INNER JOIN on every row in a archive table that can potentially have tens to hundreds of thousands of rows,
+        // so we first look for sites w/ invalidated archives, and use this as a constraint in getInvalidatedArchiveIdsSafeToDelete() below.
+        // the constraint will hit an INDEX and speed up the inner join that happens in getInvalidatedArchiveIdsSafeToDelete().
+        $idSites = $this->model->getSitesWithInvalidatedArchive($numericTable);
+        if (empty($idSites)) {
+            return;
+        }
+
+        $archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
+        if (empty($archiveIds)) {
+            return;
+        }
+
+        $this->deleteArchiveIds($date, $archiveIds);
+    }
+
+    /**
+     * Removes the outdated archives for the given month.
+     * (meaning they are marked with a done flag of ArchiveWriter::DONE_OK_TEMPORARY or ArchiveWriter::DONE_ERROR)
+     *
+     * @param Date $dateStart Only the month will be used
+     */
+    public function purgeOutdatedArchives(Date $dateStart)
+    {
+        $purgeArchivesOlderThan = $this->getOldestTemporaryArchiveToKeepThreshold();
+
+        $idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan);
+        if (!empty($idArchivesToDelete)) {
+            $this->deleteArchiveIds($dateStart, $idArchivesToDelete);
+        }
+
+        Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]",
+                   $purgeArchivesOlderThan,
+                   $dateStart->toString("Y-m"),
+                   implode(',', $idArchivesToDelete));
+    }
+
+    protected function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan)
+    {
+        $archiveTable = ArchiveTableCreator::getNumericTable($date);
+
+        $result = $this->model->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan);
+
+        $idArchivesToDelete = array();
+        if (!empty($result)) {
+            foreach ($result as $row) {
+                $idArchivesToDelete[] = $row['idarchive'];
+            }
+        }
+
+        return $idArchivesToDelete;
+    }
+
+    /**
+     * Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space.
+     *
+     * @param $date Date
+     */
+    public function purgeArchivesWithPeriodRange(Date $date)
+    {
+        $numericTable = ArchiveTableCreator::getNumericTable($date);
+        $blobTable    = ArchiveTableCreator::getBlobTable($date);
+
+        $this->model->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan);
+
+        Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]",
+            $this->purgeCustomRangesOlderThan, $numericTable);
+    }
+
+    /**
+     * Deletes by batches Archive IDs in the specified month,
+     *
+     * @param Date $date
+     * @param $idArchivesToDelete
+     */
+    protected function deleteArchiveIds(Date $date, $idArchivesToDelete)
+    {
+        $batches      = array_chunk($idArchivesToDelete, 1000);
+        $numericTable = ArchiveTableCreator::getNumericTable($date);
+        $blobTable    = ArchiveTableCreator::getBlobTable($date);
+
+        foreach ($batches as $idsToDelete) {
+            $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
+        }
+    }
+
+    /**
+     * Returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged.
+     *
+     * @return int|bool  Outdated archives older than this timestamp should be purged
+     */
+    protected function getOldestTemporaryArchiveToKeepThreshold()
+    {
+        $temporaryArchivingTimeout = Rules::getTodayArchiveTimeToLive();
+        if (Rules::isBrowserTriggerEnabled()) {
+            // If Browser Archiving is enabled, it is likely there are many more temporary archives
+            // We delete more often which is safe, since reports are re-processed on demand
+            return Date::factory($this->now - 2 * $temporaryArchivingTimeout)->getDateTime();
+        }
+
+        // If cron core:archive command is building the reports, we should keep all temporary reports from today
+        return $this->yesterday->getDateTime();
+    }
+
+    private static function getDefaultCustomRangeToPurgeAgeThreshold()
+    {
+        $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days'];
+        return Date::factory('today')->subDay($daysRangesValid)->getDateTime();
+    }
+
+    /**
+     * For tests.
+     *
+     * @param Date $yesterday
+     */
+    public function setYesterdayDate(Date $yesterday)
+    {
+        $this->yesterday = $yesterday;
+    }
+
+    /**
+     * For tests.
+     *
+     * @param Date $today
+     */
+    public function setTodayDate(Date $today)
+    {
+        $this->today = $today;
+    }
+
+    /**
+     * For tests.
+     *
+     * @param int $now
+     */
+    public function setNow($now)
+    {
+        $this->now = $now;
+    }
+}
\ No newline at end of file
diff --git a/core/ArchiveProcessor/Rules.php b/core/ArchiveProcessor/Rules.php
index 06b38aa27d..4ad2fd1b25 100644
--- a/core/ArchiveProcessor/Rules.php
+++ b/core/ArchiveProcessor/Rules.php
@@ -10,7 +10,6 @@ namespace Piwik\ArchiveProcessor;
 
 use Exception;
 use Piwik\Config;
-use Piwik\Container\StaticContainer;
 use Piwik\DataAccess\ArchiveWriter;
 use Piwik\Date;
 use Piwik\Log;
@@ -113,37 +112,6 @@ class Rules
         return $doneFlags;
     }
 
-    /**
-     * Returns false if we should not purge data for this month,
-     * or returns a timestamp indicating outdated archives older than this timestamp (processed before) can be purged.
-     *
-     * Note: when calling this function it is assumed that the callee will purge the outdated archives afterwards.
-     *
-     * @param \Piwik\Date $date
-     * @return int|bool  Outdated archives older than this timestamp should be purged
-     */
-    public static function shouldPurgeOutdatedArchives(Date $date)
-    {
-        // we only delete archives if we are able to process them, otherwise, the browser might process reports
-        // when &segment= is specified (or custom date range) and would below, delete temporary archives that the
-        // browser is not able to process until next cron run (which could be more than 1 hour away)
-        if (! self::isRequestAuthorizedToArchive()){
-            Log::info("Purging temporary archives: skipped (no authorization)");
-            return false;
-        }
-
-        $temporaryArchivingTimeout = self::getTodayArchiveTimeToLive();
-
-        if (self::isBrowserTriggerEnabled()) {
-            // If Browser Archiving is enabled, it is likely there are many more temporary archives
-            // We delete more often which is safe, since reports are re-processed on demand
-            return Date::factory(time() - 2 * $temporaryArchivingTimeout)->getDateTime();
-        } 
-
-        // If cron core:archive command is building the reports, we should keep all temporary reports from today
-        return Date::factory('yesterday')->getDateTime();
-    }
-
     public static function getMinTimeProcessedForTemporaryArchive(
         Date $dateStart, \Piwik\Period $period, Segment $segment, Site $site)
     {
@@ -309,5 +277,4 @@ class Rules
 
         return $possibleValues;
     }
-
-}
+}
\ No newline at end of file
diff --git a/core/Concurrency/DistributedList.php b/core/Concurrency/DistributedList.php
new file mode 100644
index 0000000000..eecfab6a55
--- /dev/null
+++ b/core/Concurrency/DistributedList.php
@@ -0,0 +1,138 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Concurrency;
+
+use Piwik\Option;
+
+/**
+ * Manages a simple distributed list stored in an Option. No locking occurs, so the list
+ * is not thread safe, and should only be used for use cases where atomicity is not
+ * important.
+ *
+ * The list of items is serialized and stored in an Option. Items are converted to string
+ * before being persisted, so it is not expected to unserialize objects.
+ */
+class DistributedList
+{
+    /**
+     * The name of the option to store the list in.
+     *
+     * @var string
+     */
+    private $optionName;
+
+    /**
+     * Constructor.
+     *
+     * @param string $optionName
+     */
+    public function __construct($optionName)
+    {
+        $this->optionName = $optionName;
+    }
+
+    /**
+     * Queries the option table and returns all items in this list.
+     *
+     * @return array
+     */
+    public function getAll()
+    {
+        Option::clearCachedOption($this->optionName);
+        $array = Option::get($this->optionName);
+
+        if ($array
+            && ($array = unserialize($array))
+            && count($array)
+        ) {
+            return $array;
+        }
+        return array();
+    }
+
+    /**
+     * Sets the contents of the list in the option table.
+     *
+     * @param string[] $items
+     */
+    public function setAll($items)
+    {
+        foreach ($items as &$item) {
+            $item = (string)$item;
+        }
+
+        Option::set($this->optionName, serialize($items));
+    }
+
+    /**
+     * Adds one or more items to the list in the option table.
+     *
+     * @param string|array $item
+     */
+    public function add($item)
+    {
+        $allItems = $this->getAll();
+        if (is_array($item)) {
+            $allItems = array_merge($allItems, $item);
+        } else {
+            $allItems[] = $item;
+        }
+
+        $this->setAll($allItems);
+    }
+
+    /**
+     * Removes one or more items by value from the list in the option table.
+     *
+     * Does not preserve array keys.
+     *
+     * @param string|array $items
+     */
+    public function remove($items)
+    {
+        if (!is_array($items)) {
+            $items = array($items);
+        }
+
+        $allItems = $this->getAll();
+
+        foreach ($items as $item) {
+            $existingIndex = array_search($item, $allItems);
+            if ($existingIndex === false) {
+                return;
+            }
+
+            unset($allItems[$existingIndex]);
+        }
+
+        $this->setAll(array_values($allItems));
+    }
+
+    /**
+     * Removes one or more items by index from the list in the option table.
+     *
+     * Does not preserve array keys.
+     *
+     * @param int[]|int $indices
+     */
+    public function removeByIndex($indices)
+    {
+        if (!is_array($indices)) {
+            $indices = array($indices);
+        }
+
+        $indices = array_unique($indices);
+
+        $allItems = $this->getAll();
+        foreach ($indices as $index) {
+            unset($allItems[$index]);
+        }
+
+        $this->setAll(array_values($allItems));
+    }
+}
\ No newline at end of file
diff --git a/core/Console.php b/core/Console.php
index fac27d8036..cfed85a994 100644
--- a/core/Console.php
+++ b/core/Console.php
@@ -13,6 +13,7 @@ use Piwik\Container\StaticContainer;
 use Piwik\Plugin\Manager as PluginManager;
 use Symfony\Bridge\Monolog\Handler\ConsoleHandler;
 use Symfony\Component\Console\Application;
+use Symfony\Component\Console\Command\Command;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
@@ -65,10 +66,16 @@ class Console extends Application
     {
         if (!class_exists($command)) {
             Log::warning(sprintf('Cannot add command %s, class does not exist', $command));
-        } elseif (!is_subclass_of($command, 'Piwik\Plugin\ConsoleCommand')) {
+        } else if (!is_subclass_of($command, 'Piwik\Plugin\ConsoleCommand')) {
             Log::warning(sprintf('Cannot add command %s, class does not extend Piwik\Plugin\ConsoleCommand', $command));
         } else {
-            $this->add(new $command);
+            /** @var Command $commandInstance */
+            $commandInstance = new $command;
+
+            // do not add the command if it already exists; this way we can add the command ourselves in tests
+            if (!$this->has($commandInstance->getName())) {
+                $this->add($commandInstance);
+            }
         }
     }
 
diff --git a/core/CronArchive.php b/core/CronArchive.php
index 29c897c8bf..0ff2ee8242 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -12,11 +12,11 @@ use Exception;
 use Piwik\ArchiveProcessor\Rules;
 use Piwik\CronArchive\FixedSiteIds;
 use Piwik\CronArchive\SharedSiteIds;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Metrics\Formatter;
 use Piwik\Period\Factory as PeriodFactory;
-use Piwik\DataAccess\InvalidatedReports;
+use Piwik\CronArchive\SitesToReprocessDistributedList;
 use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI;
 use Piwik\Plugins\SitesManager\API as APISitesManager;
 use Piwik\Plugins\UsersManager\API as APIUsersManager;
@@ -542,8 +542,8 @@ class CronArchive
         if(!$success) {
             // cancel marking the site as reprocessed
             if($websiteInvalidatedShouldReprocess) {
-                $store = new InvalidatedReports();
-                $store->addInvalidatedSitesToReprocess(array($idSite));
+                $store = new SitesToReprocessDistributedList();
+                $store->add($idSite);
             }
         }
 
@@ -657,8 +657,8 @@ class CronArchive
         // does not archive the same idSite
         $websiteInvalidatedShouldReprocess = $this->isOldReportInvalidatedForWebsite($idSite);
         if ($websiteInvalidatedShouldReprocess) {
-            $store = new InvalidatedReports();
-            $store->storeSiteIsReprocessed($idSite);
+            $store = new SitesToReprocessDistributedList();
+            $store->remove($idSite);
         }
 
         // when some data was purged from this website
@@ -685,8 +685,8 @@ class CronArchive
 
             // cancel marking the site as reprocessed
             if($websiteInvalidatedShouldReprocess) {
-                $store = new InvalidatedReports();
-                $store->addInvalidatedSitesToReprocess(array($idSite));
+                $store = new SitesToReprocessDistributedList();
+                $store->add($idSite);
             }
 
             $this->logError("Empty or invalid response '$content' for website id $idSite, " . $timerWebsite->__toString() . ", skipping");
@@ -1083,8 +1083,8 @@ class CronArchive
 
     private function updateIdSitesInvalidatedOldReports()
     {
-        $store = new InvalidatedReports();
-        $this->idSitesInvalidatedOldReports = $store->getSitesToReprocess();
+        $store = new SitesToReprocessDistributedList();
+        $this->idSitesInvalidatedOldReports = $store->getAll();
     }
 
     /**
diff --git a/core/CronArchive/SitesToReprocessDistributedList.php b/core/CronArchive/SitesToReprocessDistributedList.php
new file mode 100644
index 0000000000..44b1eedca6
--- /dev/null
+++ b/core/CronArchive/SitesToReprocessDistributedList.php
@@ -0,0 +1,40 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\CronArchive;
+
+use Piwik\Concurrency\DistributedList;
+
+/**
+ * Distributed list that stores the list of IDs of sites whose archives should be reprocessed.
+ *
+ * CronArchive will read this list of sites when archiving is being run, and make sure the sites
+ * are re-archived.
+ *
+ * Any class/API method/command/etc. is allowed to add site IDs to this list.
+ */
+class SitesToReprocessDistributedList extends DistributedList
+{
+    const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds';
+
+    public function __construct()
+    {
+        parent::__construct(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function setAll($items)
+    {
+        $items = array_unique($items);
+        $items = array_values($items);
+
+        parent::setAll($items);
+    }
+}
\ No newline at end of file
diff --git a/core/DataAccess/ArchivePurger.php b/core/DataAccess/ArchivePurger.php
deleted file mode 100644
index e7f185f475..0000000000
--- a/core/DataAccess/ArchivePurger.php
+++ /dev/null
@@ -1,139 +0,0 @@
-<?php
-/**
- * Piwik - free/libre analytics platform
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- *
- */
-namespace Piwik\DataAccess;
-
-use Exception;
-use Piwik\ArchiveProcessor\Rules;
-use Piwik\Config;
-use Piwik\Date;
-use Piwik\Db;
-use Piwik\Log;
-use Piwik\Piwik;
-
-/**
- *
- * This class purges two types of archives:
- *
- * (1) Deletes invalidated archives (from ArchiveInvalidator)
- *
- * (2) Deletes outdated archives (the temporary or errored archives)
- *
- *
- * @package Piwik\DataAccess
- */
-class ArchivePurger
-{
-    public static function purgeInvalidatedArchives()
-    {
-        $store = new InvalidatedReports();
-        $idSitesByYearMonth = $store->getSitesByYearMonthArchiveToPurge();
-        foreach ($idSitesByYearMonth as $yearMonth => $idSites) {
-            if(empty($idSites)) {
-                continue;
-            }
-
-            $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01');
-            $numericTable = ArchiveTableCreator::getNumericTable($date);
-
-            $archiveIds = self::getModel()->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
-
-            if (count($archiveIds) == 0) {
-                continue;
-            }
-            self::deleteArchiveIds($date, $archiveIds);
-
-            $store->markSiteIdsHaveBeenPurged($idSites, $yearMonth);
-        }
-    }
-
-    /**
-     * Removes the outdated archives for the given month.
-     * (meaning they are marked with a done flag of ArchiveWriter::DONE_OK_TEMPORARY or ArchiveWriter::DONE_ERROR)
-     *
-     * @param Date $dateStart Only the month will be used
-     */
-    public static function purgeOutdatedArchives(Date $dateStart)
-    {
-        $purgeArchivesOlderThan = Rules::shouldPurgeOutdatedArchives($dateStart);
-
-        if (!$purgeArchivesOlderThan) {
-            return;
-        }
-
-        $idArchivesToDelete = self::getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan);
-
-        if (!empty($idArchivesToDelete)) {
-            self::deleteArchiveIds($dateStart, $idArchivesToDelete);
-        }
-
-        self::deleteArchivesWithPeriodRange($dateStart);
-
-        Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]",
-                   $purgeArchivesOlderThan,
-                   $dateStart->toString("Y-m"),
-                   implode(',', $idArchivesToDelete));
-    }
-
-    protected static function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan)
-    {
-        $archiveTable = ArchiveTableCreator::getNumericTable($date);
-
-        $result = self::getModel()->getTemporaryArchivesOlderThan($archiveTable, $purgeArchivesOlderThan);
-
-        $idArchivesToDelete = array();
-        if (!empty($result)) {
-            foreach ($result as $row) {
-                $idArchivesToDelete[] = $row['idarchive'];
-            }
-        }
-
-        return $idArchivesToDelete;
-    }
-
-    /**
-     * Deleting "Custom Date Range" reports after 1 day, since they can be re-processed and would take up un-necessary space.
-     *
-     * @param $date Date
-     */
-    protected static function deleteArchivesWithPeriodRange(Date $date)
-    {
-        $numericTable = ArchiveTableCreator::getNumericTable($date);
-        $blobTable    = ArchiveTableCreator::getBlobTable($date);
-        $daysRangesValid = Config::getInstance()->General['purge_date_range_archives_after_X_days'];
-        $pastDate    = Date::factory('today')->subDay($daysRangesValid)->getDateTime();
-
-        self::getModel()->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $pastDate);
-
-        Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]",
-            $pastDate, $numericTable);
-    }
-
-    /**
-     * Deletes by batches Archive IDs in the specified month,
-     *
-     * @param Date $date
-     * @param $idArchivesToDelete
-     */
-    protected static function deleteArchiveIds(Date $date, $idArchivesToDelete)
-    {
-        $batches      = array_chunk($idArchivesToDelete, 1000);
-        $numericTable = ArchiveTableCreator::getNumericTable($date);
-        $blobTable    = ArchiveTableCreator::getBlobTable($date);
-
-        foreach ($batches as $idsToDelete) {
-            self::getModel()->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
-        }
-    }
-
-    private static function getModel()
-    {
-        return new Model();
-    }
-
-}
diff --git a/core/DataAccess/InvalidatedReports.php b/core/DataAccess/InvalidatedReports.php
deleted file mode 100644
index 64f863e0ad..0000000000
--- a/core/DataAccess/InvalidatedReports.php
+++ /dev/null
@@ -1,168 +0,0 @@
-<?php
-/**
- * Piwik - free/libre analytics platform
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- *
- */
-
-namespace Piwik\DataAccess;
-
-use Piwik\Option;
-
-/**
- * Keeps track of which reports were invalidated via CoreAdminHome.invalidateArchivedReports API.
- *
- * This is used by:
- *
- * 1. core:archive command to know which websites should be reprocessed
- *
- * 2. scheduled task purgeInvalidatedArchives to know which websites/months should be purged
- *
- */
-class InvalidatedReports
-{
-    const OPTION_INVALIDATED_IDSITES_TO_REPROCESS = 'InvalidatedOldReports_WebsiteIds';
-    const OPTION_INVALIDATED_DATES_SITES_TO_PURGE = 'InvalidatedOldReports_DatesWebsiteIds';
-
-    /**
-     * Mark the sites IDs and Dates as being invalidated, so we can purge them later on.
-     *
-     * @param array $idSites
-     * @param array $yearMonths
-     */
-    public function addSitesToPurgeForYearMonths(array $idSites, $yearMonths)
-    {
-        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
-
-        foreach($yearMonths as $yearMonthToPurge) {
-
-            if(isset($idSitesByYearMonth[$yearMonthToPurge])) {
-                $existingIdSitesToPurge = $idSitesByYearMonth[$yearMonthToPurge];
-                $idSites = array_merge($existingIdSitesToPurge, $idSites);
-                $idSites = array_unique($idSites);
-            }
-            $idSitesByYearMonth[$yearMonthToPurge] = $idSites;
-        }
-        $this->persistSitesByYearMonthToPurge($idSitesByYearMonth);
-    }
-
-    /**
-     * Returns the list of websites IDs for which invalidated archives can be purged.
-     */
-    public function getSitesByYearMonthArchiveToPurge()
-    {
-        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
-
-        // From this list we remove the websites that are not yet re-processed
-        // so we don't purge them before they were re-processed
-        $idSitesNotYetReprocessed = $this->getSitesToReprocess();
-
-        foreach($idSitesByYearMonth as $yearMonth => &$idSites) {
-            $idSites = array_diff($idSites, $idSitesNotYetReprocessed);
-        }
-        return $idSitesByYearMonth;
-    }
-
-    public function markSiteIdsHaveBeenPurged(array $idSites, $yearMonth)
-    {
-        $idSitesByYearMonth = $this->getSitesByYearMonthToPurge();
-
-        if(!isset($idSitesByYearMonth[$yearMonth])) {
-            return;
-        }
-
-        $idSitesByYearMonth[$yearMonth] = array_diff($idSitesByYearMonth[$yearMonth], $idSites);
-        $this->persistSitesByYearMonthToPurge($idSitesByYearMonth);
-    }
-
-    /**
-     * Record those website IDs as having been invalidated
-     *
-     * @param $idSites
-     */
-    public function addInvalidatedSitesToReprocess(array $idSites)
-    {
-        $siteIdsToReprocess = $this->getSitesToReprocess();
-        $siteIdsToReprocess = array_merge($siteIdsToReprocess, $idSites);
-        $this->setSitesToReprocess($siteIdsToReprocess);
-    }
-
-
-    /**
-     * @param $idSite
-     */
-    public function storeSiteIsReprocessed($idSite)
-    {
-        $siteIdsToReprocess = $this->getSitesToReprocess();
-
-        if (count($siteIdsToReprocess)) {
-            $found = array_search($idSite, $siteIdsToReprocess);
-            if ($found !== false) {
-                unset($siteIdsToReprocess[$found]);
-                $this->setSitesToReprocess($siteIdsToReprocess);
-            }
-        }
-    }
-
-    /**
-     * Returns array of idSites to force re-process next time core:archive command runs
-     *
-     * @return array of id sites
-     */
-    public function getSitesToReprocess()
-    {
-        return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS);
-    }
-
-    /**
-     * @return array|false|mixed|string
-     */
-    private function getSitesByYearMonthToPurge()
-    {
-        return $this->getArrayValueFromOptionName(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE);
-    }
-
-    /**
-     * @param $websiteIdsInvalidated
-     */
-    private function setSitesToReprocess($websiteIdsInvalidated)
-    {
-        $websiteIdsInvalidated = array_unique($websiteIdsInvalidated);
-        $websiteIdsInvalidated = array_values($websiteIdsInvalidated);
-        Option::set(self::OPTION_INVALIDATED_IDSITES_TO_REPROCESS, serialize($websiteIdsInvalidated));
-    }
-
-    /**
-     * @param $optionName
-     * @return array|false|mixed|string
-     */
-    private function getArrayValueFromOptionName($optionName)
-    {
-        Option::clearCachedOption($optionName);
-        $array = Option::get($optionName);
-
-        if ($array
-            && ($array = unserialize($array))
-            && count($array)
-        ) {
-            return $array;
-        }
-        return array();
-    }
-
-    /**
-     * @param $idSitesByYearMonth
-     */
-    private function persistSitesByYearMonthToPurge($idSitesByYearMonth)
-    {
-        // remove dates for which there are no sites to purge
-        $idSitesByYearMonth = array_filter($idSitesByYearMonth);
-
-        Option::set(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE, serialize($idSitesByYearMonth));
-    }
-
-
-
-}
\ No newline at end of file
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index 239bce6a23..0980983fe3 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -246,6 +246,23 @@ class Model
     }
 
     /**
+     * Returns the site IDs for invalidated archives in an archive table.
+     *
+     * @param string $numericTable The numeric table to search through.
+     * @return int[]
+     */
+    public function getSitesWithInvalidatedArchive($numericTable)
+    {
+        $rows = Db::fetchAll("SELECT DISTINCT idsite FROM `$numericTable` WHERE name LIKE 'done%' AND value = " . ArchiveWriter::DONE_INVALIDATED);
+
+        $result = array();
+        foreach ($rows as $row) {
+            $result[] = $row['idsite'];
+        }
+        return $result;
+    }
+
+    /**
      * Returns the SQL condition used to find successfully completed archives that
      * this instance is querying for.
      */
diff --git a/core/Tracker/Visit.php b/core/Tracker/Visit.php
index e5dbc69ef6..f672cd96da 100644
--- a/core/Tracker/Visit.php
+++ b/core/Tracker/Visit.php
@@ -11,7 +11,7 @@ namespace Piwik\Tracker;
 
 use Piwik\Common;
 use Piwik\Config;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Date;
 use Piwik\Exception\UnexpectedWebsiteFoundException;
 use Piwik\Network\IPUtils;
diff --git a/plugins/CoreAdminHome/API.php b/plugins/CoreAdminHome/API.php
index 60ba204d45..6329fc8993 100644
--- a/plugins/CoreAdminHome/API.php
+++ b/plugins/CoreAdminHome/API.php
@@ -10,7 +10,7 @@ namespace Piwik\Plugins\CoreAdminHome;
 
 use Exception;
 use Piwik\Container\StaticContainer;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Db;
 use Piwik\Piwik;
 use Piwik\Scheduler\Scheduler;
diff --git a/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php b/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php
index 47a058c7ad..be9c74a152 100644
--- a/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php
+++ b/plugins/CoreAdminHome/Commands/FixDuplicateLogActions.php
@@ -11,7 +11,7 @@ namespace Piwik\Plugins\CoreAdminHome\Commands;
 use Piwik\Common;
 use Piwik\Container\StaticContainer;
 use Piwik\DataAccess\Actions;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Plugin\ConsoleCommand;
 use Piwik\Plugins\CoreAdminHome\Model\DuplicateActionRemover;
 use Piwik\Timer;
diff --git a/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php
new file mode 100644
index 0000000000..89696eaa9b
--- /dev/null
+++ b/plugins/CoreAdminHome/Commands/PurgeOldArchiveData.php
@@ -0,0 +1,180 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\CoreAdminHome\Commands;
+
+use Piwik\Archive;
+use Piwik\Archive\ArchivePurger;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Plugin\ConsoleCommand;
+use Piwik\Timer;
+use Symfony\Component\Console\Input\InputArgument;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * Command that allows users to force purge old or invalid archive data. In the event of a failure
+ * in the archive purging scheduled task, this command can be used to manually delete old/invalid archives.
+ */
+class PurgeOldArchiveData extends ConsoleCommand
+{
+    const ALL_DATES_STRING = 'all';
+
+    /**
+     * @var ArchivePurger
+     */
+    private $archivePurger;
+
+    /**
+     * For tests.
+     *
+     * @var Date
+     */
+    public static $todayOverride = null;
+
+    public function __construct(ArchivePurger $archivePurger = null)
+    {
+        parent::__construct();
+
+        $this->archivePurger = $archivePurger ?: new ArchivePurger();
+    }
+
+    protected function configure()
+    {
+        $this->setName('core:purge-old-archive-data');
+        $this->setDescription('Purges out of date and invalid archive data from archive tables.');
+        $this->addArgument("dates", InputArgument::IS_ARRAY | InputArgument::OPTIONAL,
+            "The months of the archive tables to purge data from. By default, only deletes from the current month. Use '" . self::ALL_DATES_STRING. "' for all dates.",
+            array(self::getToday()->toString()));
+        $this->addOption('exclude-outdated', null, InputOption::VALUE_NONE, "Do not purge outdated archive data.");
+        $this->addOption('exclude-invalidated', null, InputOption::VALUE_NONE, "Do not purge invalidated archive data.");
+        $this->addOption('exclude-ranges', null, InputOption::VALUE_NONE, "Do not purge custom ranges.");
+        $this->addOption('skip-optimize-tables', null, InputOption::VALUE_NONE, "Do not run OPTIMIZE TABLES query on affected archive tables.");
+        $this->setHelp("By default old and invalidated archives are purged. Custom ranges are also purged with outdated archives.\n\n"
+                     . "Note: archive purging is done during scheduled task execution, so under normal circumstances, you should not need to "
+                     . "run this command manually.");
+
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output)
+    {
+        $archivePurger = $this->archivePurger;
+
+        $dates = $this->getDatesToPurgeFor($input);
+
+        $excludeOutdated = $input->getOption('exclude-outdated');
+        if ($excludeOutdated) {
+            $output->writeln("Skipping purge outdated archive data.");
+        } else {
+            foreach ($dates as $date) {
+                $message = sprintf("Purging outdated archives for %s...", $date->toString('Y_m'));
+                $this->performTimedPurging($output, $message, function () use ($date, $archivePurger) {
+                    $archivePurger->purgeOutdatedArchives($date);
+                });
+            }
+        }
+
+        $excludeInvalidated = $input->getOption('exclude-invalidated');
+        if ($excludeInvalidated) {
+            $output->writeln("Skipping purge invalidated archive data.");
+        } else {
+            foreach ($dates as $date) {
+                $message = sprintf("Purging invalidated archives for %s...", $date->toString('Y_m'));
+                $this->performTimedPurging($output, $message, function () use ($archivePurger, $date) {
+                    $archivePurger->purgeInvalidatedArchivesFrom($date);
+                });
+            }
+        }
+
+        $excludeCustomRanges = $input->getOption('exclude-ranges');
+        if ($excludeCustomRanges) {
+            $output->writeln("Skipping purge custom range archives.");
+        } else {
+            foreach ($dates as $date) {
+                $message = sprintf("Purging custom range archives for %s...", $date->toString('Y_m'));
+                $this->performTimedPurging($output, $message, function () use ($date, $archivePurger) {
+                    $archivePurger->purgeArchivesWithPeriodRange($date);
+                });
+            }
+        }
+
+        $skipOptimizeTables = $input->getOption('skip-optimize-tables');
+        if ($skipOptimizeTables) {
+            $output->writeln("Skipping OPTIMIZE TABLES.");
+        } else {
+            $this->optimizeArchiveTables($output, $dates);
+        }
+    }
+
+    /**
+     * @param InputInterface $input
+     * @return Date[]
+     */
+    private function getDatesToPurgeFor(InputInterface $input)
+    {
+        $dates = array();
+
+        $dateSpecifier = $input->getArgument('dates');
+        if (count($dateSpecifier) === 1
+            && reset($dateSpecifier) == self::ALL_DATES_STRING
+        ) {
+            foreach (ArchiveTableCreator::getTablesArchivesInstalled() as $table) {
+                $tableDate = ArchiveTableCreator::getDateFromTableName($table);
+
+                list($year, $month) = explode('_', $tableDate);
+
+                $dates[] = Date::factory($year . '-' . $month . '-' . '01');
+            }
+        } else {
+            foreach ($dateSpecifier as $date) {
+                $dates[] = Date::factory($date);
+            }
+        }
+
+        return $dates;
+    }
+
+    private function performTimedPurging(OutputInterface $output, $startMessage, $callback)
+    {
+        $timer = new Timer();
+
+        $output->write($startMessage);
+
+        $callback();
+
+        $output->writeln("Done. <comment>[" . $timer->__toString() . "]</comment>");
+    }
+
+    /**
+     * @param Date[] $dates
+     */
+    private function optimizeArchiveTables(OutputInterface $output, $dates)
+    {
+        $output->writeln("Optimizing archive tables...");
+
+        foreach ($dates as $date) {
+            $numericTable = ArchiveTableCreator::getNumericTable($date);
+            $this->performTimedPurging($output, "Optimizing table $numericTable...", function () use ($numericTable) {
+                Db::optimizeTables($numericTable);
+            });
+
+            $blobTable = ArchiveTableCreator::getBlobTable($date);
+            $this->performTimedPurging($output, "Optimizing table $blobTable...", function () use ($blobTable) {
+                Db::optimizeTables($blobTable);
+            });
+        }
+    }
+
+    private static function getToday()
+    {
+        return self::$todayOverride ?: Date::today();
+    }
+}
\ No newline at end of file
diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php
index d633b9fd5d..700c7d69c2 100644
--- a/plugins/CoreAdminHome/Tasks.php
+++ b/plugins/CoreAdminHome/Tasks.php
@@ -8,13 +8,28 @@
  */
 namespace Piwik\Plugins\CoreAdminHome;
 
-use Piwik\DataAccess\ArchivePurger;
+use Piwik\ArchiveProcessor\Rules;
+use Piwik\Archive\ArchivePurger;
+use Piwik\Container\StaticContainer;
 use Piwik\DataAccess\ArchiveTableCreator;
 use Piwik\Date;
 use Piwik\Db;
+use Piwik\Log;
+use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
+use Piwik\SettingsServer;
 
 class Tasks extends \Piwik\Plugin\Tasks
 {
+    /**
+     * @var ArchivePurger
+     */
+    private $archivePurger;
+
+    public function __construct(ArchivePurger $archivePurger = null)
+    {
+        $this->archivePurger = $archivePurger ?: new ArchivePurger();
+    }
+
     public function schedule()
     {
         // general data purge on older archive tables, executed daily
@@ -29,21 +44,47 @@ class Tasks extends \Piwik\Plugin\Tasks
 
     public function purgeOutdatedArchives()
     {
+        $logger = StaticContainer::get('Psr\Log\LoggerInterface');
+
+        if ($this->willPurgingCausePotentialProblemInUI()) {
+            $logger->info("Purging temporary archives: skipped (browser triggered archiving not enabled & not running after core:archive)");
+            return false;
+        }
+
         $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
+
+        $logger->info("Purging archives in {tableCount} archive tables.", array('tableCount' => count($archiveTables)));
+
+        // keep track of dates we purge for, since getTablesArchivesInstalled() will return numeric & blob
+        // tables (so dates will appear two times, and we should only purge once per date)
+        $datesPurged = array();
+
         foreach ($archiveTables as $table) {
             $date = ArchiveTableCreator::getDateFromTableName($table);
             list($year, $month) = explode('_', $date);
 
             // Somehow we may have archive tables created with older dates, prevent exception from being thrown
-            if ($year > 1990) {
-                ArchivePurger::purgeOutdatedArchives(Date::factory("$year-$month-15"));
+            if ($year > 1990
+                && empty($datesPurged[$date])
+            ) {
+                $dateObj = Date::factory("$year-$month-15");
+
+                $this->archivePurger->purgeOutdatedArchives($dateObj);
+                $this->archivePurger->purgeArchivesWithPeriodRange($dateObj);
+
+                $datesPurged[$date] = true;
             }
         }
     }
 
     public function purgeInvalidatedArchives()
     {
-        ArchivePurger::purgeInvalidatedArchives();
+        $archivesToPurge = new ArchivesToPurgeDistributedList();
+        foreach ($archivesToPurge->getAllAsDates() as $date) {
+            $this->archivePurger->purgeInvalidatedArchivesFrom($date);
+
+            $archivesToPurge->removeDate($date);
+        }
     }
 
     public function optimizeArchiveTable()
@@ -51,4 +92,18 @@ class Tasks extends \Piwik\Plugin\Tasks
         $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
         Db::optimizeTables($archiveTables);
     }
+
+    /**
+     * we should only purge outdated & custom range archives if we know cron archiving has just run,
+     * or if browser triggered archiving is enabled. if cron archiving has run, then we know the latest
+     * archives are in the database, and we can remove temporary ones. if browser triggered archiving is
+     * enabled, then we know any archives that are wrongly purged, can be re-archived on demand.
+     * this prevents some situations where "no data" is displayed for reports that should have data.
+     *
+     * @return bool
+     */
+    private function willPurgingCausePotentialProblemInUI()
+    {
+        return !Rules::isRequestAuthorizedToArchive();
+    }
 }
\ No newline at end of file
diff --git a/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php
new file mode 100644
index 0000000000..6d56286632
--- /dev/null
+++ b/plugins/CoreAdminHome/Tasks/ArchivesToPurgeDistributedList.php
@@ -0,0 +1,64 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Plugins\CoreAdminHome\Tasks;
+
+use Piwik\Concurrency\DistributedList;
+use Piwik\Date;
+
+/**
+ * Distributed list that holds a list of year-month archive table identifiers (eg, 2015_01 or 2014_11). Each item in the
+ * list is expected to identify a pair of archive tables that contain invalidated archives.
+ *
+ * The archiving purging scheduled task will read items in this list when executing the daily purge.
+ *
+ * This class is necessary in order to keep the archive purging scheduled task fast. W/o a way to keep track of
+ * tables w/ invalid data, the task would have to iterate over every table, which is not desired for a task that
+ * is executed daily.
+ *
+ * If users find other tables contain invalidated archives, they can use the core:purge-old-archive-data command
+ * to manually purge them.
+ */
+class ArchivesToPurgeDistributedList extends DistributedList
+{
+    const OPTION_INVALIDATED_DATES_SITES_TO_PURGE = 'InvalidatedOldReports_DatesWebsiteIds';
+
+    public function __construct()
+    {
+        parent::__construct(self::OPTION_INVALIDATED_DATES_SITES_TO_PURGE);
+    }
+
+    /**
+     * @inheritdoc
+     */
+    public function setAll($yearMonths)
+    {
+        $yearMonths = array_unique($yearMonths);
+        parent::setAll($yearMonths);
+    }
+
+    public function getAllAsDates()
+    {
+        $dates = array();
+        foreach ($this->getAll() as $yearMonth) {
+            try {
+                $date = Date::factory(str_replace('_', '-', $yearMonth) . '-01');
+            } catch (\Exception $ex) {
+                continue; // invalid year month in distributed list
+            }
+
+            $dates[] = $date;
+        }
+        return $dates;
+    }
+
+    public function removeDate(Date $date)
+    {
+        $yearMonth = $date->toString('Y_m');
+        $this->remove($yearMonth);
+    }
+}
\ No newline at end of file
diff --git a/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php b/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php
new file mode 100644
index 0000000000..ecb07aa438
--- /dev/null
+++ b/plugins/CoreAdminHome/tests/Integration/Commands/PurgeOldArchiveDataTest.php
@@ -0,0 +1,152 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Plugins\CoreAdminHome\tests\Integration\Commands;
+
+use Piwik\Archive\ArchivePurger;
+use Piwik\Console;
+use Piwik\Date;
+use Piwik\Plugins\CoreAdminHome\Commands\PurgeOldArchiveData;
+use Piwik\Tests\Fixtures\RawArchiveDataWithTempAndInvalidated;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+use Symfony\Component\Console\Tester\ApplicationTester;
+
+/**
+ * @group Core
+ */
+class PurgeOldArchiveDataTest extends IntegrationTestCase
+{
+    /**
+     * @var RawArchiveDataWithTempAndInvalidated
+     */
+    public static $fixture = null;
+
+    /**
+     * @var ApplicationTester
+     */
+    protected $applicationTester = null;
+
+    /**
+     * @var Console
+     */
+    protected $application;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        PurgeOldArchiveData::$todayOverride = Date::factory('2015-02-27');
+
+        $archivePurger = new ArchivePurger();
+        $archivePurger->setTodayDate(Date::factory('2015-02-27'));
+        $archivePurger->setYesterdayDate(Date::factory('2015-02-26'));
+        $archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp());
+
+        $this->application = new Console();
+        $this->application->setAutoExit(false);
+        $this->application->add(new PurgeOldArchiveData($archivePurger));
+
+        $this->applicationTester = new ApplicationTester($this->application);
+
+        // assert the test data was setup correctly
+        self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->january);
+        self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->february);
+    }
+
+    public function tearDown()
+    {
+        PurgeOldArchiveData::$todayOverride = null;
+
+        parent::tearDown();
+    }
+
+    public function test_ExecutingCommandWithAllDates_PurgesAllExistingArchiveTables()
+    {
+        $result = $this->applicationTester->run(array(
+            'command' => 'core:purge-old-archive-data',
+            'dates' => array('all'),
+            '-vvv' => true
+        ));
+
+        $this->assertEquals(0, $result, $this->getCommandDisplayOutputErrorMessage());
+
+        self::$fixture->assertInvalidatedArchivesPurged(self::$fixture->february);
+        self::$fixture->assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled = true, self::$fixture->february);
+        self::$fixture->assertCustomRangesPurged(self::$fixture->february);
+
+        self::$fixture->assertInvalidatedArchivesPurged(self::$fixture->january);
+        self::$fixture->assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled = true, self::$fixture->january);
+        self::$fixture->assertCustomRangesPurged(self::$fixture->january);
+    }
+
+    public function test_ExecutingCommandWithNoDate_PurgesArchiveTableForToday()
+    {
+        $result = $this->applicationTester->run(array(
+            'command' => 'core:purge-old-archive-data',
+            '-vvv' => true
+        ));
+
+        $this->assertEquals(0, $result, $this->getCommandDisplayOutputErrorMessage());
+
+        self::$fixture->assertInvalidatedArchivesPurged(self::$fixture->february);
+        self::$fixture->assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled = true, self::$fixture->february);
+        self::$fixture->assertCustomRangesPurged(self::$fixture->february);
+
+        self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->january);
+        self::$fixture->assertTemporaryArchivesNotPurged(self::$fixture->january);
+        self::$fixture->assertCustomRangesNotPurged(self::$fixture->january);
+    }
+
+    public function test_ExecutingCommandWithSpecificDate_PurgesArchiveTableForDate()
+    {
+        $result = $this->applicationTester->run(array(
+            'command' => 'core:purge-old-archive-data',
+            'dates' => array('2015-01-14'),
+            '-vvv' => true
+        ));
+
+        $this->assertEquals(0, $result, $this->getCommandDisplayOutputErrorMessage());
+
+        self::$fixture->assertInvalidatedArchivesPurged(self::$fixture->january);
+        self::$fixture->assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled = true, self::$fixture->january);
+        self::$fixture->assertCustomRangesPurged(self::$fixture->january);
+
+        self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->february);
+        self::$fixture->assertTemporaryArchivesNotPurged(self::$fixture->february);
+        self::$fixture->assertCustomRangesNotPurged(self::$fixture->february);
+    }
+
+    public function test_ExecutingCommandWithExcludeOptions_SkipsAppropriatePurging()
+    {
+        $result = $this->applicationTester->run(array(
+            'command' => 'core:purge-old-archive-data',
+            'dates' => array('2015-01-14'),
+            '--exclude-outdated' => true,
+            '--exclude-invalidated' => true,
+            '--exclude-ranges' => true,
+            '--skip-optimize-tables' => true,
+            '-vvv' => true
+        ));
+
+        $this->assertEquals(0, $result, $this->getCommandDisplayOutputErrorMessage());
+
+        self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->january);
+        self::$fixture->assertTemporaryArchivesNotPurged(self::$fixture->january);
+        self::$fixture->assertCustomRangesNotPurged(self::$fixture->january);
+
+        $this->assertContains("Skipping purge outdated archive data.", $this->applicationTester->getDisplay());
+        $this->assertContains("Skipping purge invalidated archive data.", $this->applicationTester->getDisplay());
+        $this->assertContains("Skipping OPTIMIZE TABLES.", $this->applicationTester->getDisplay());
+    }
+
+    protected function getCommandDisplayOutputErrorMessage()
+    {
+        return "Command did not behave as expected. Command output: " . $this->applicationTester->getDisplay();
+    }
+}
+
+PurgeOldArchiveDataTest::$fixture = new RawArchiveDataWithTempAndInvalidated();
\ No newline at end of file
diff --git a/plugins/CoreAdminHome/tests/Integration/TasksTest.php b/plugins/CoreAdminHome/tests/Integration/TasksTest.php
new file mode 100644
index 0000000000..da3800e9c1
--- /dev/null
+++ b/plugins/CoreAdminHome/tests/Integration/TasksTest.php
@@ -0,0 +1,88 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Plugins\CoreAdminHome\tests\Integration;
+
+use Piwik\Archive\ArchivePurger;
+use Piwik\Date;
+use Piwik\Plugins\CoreAdminHome\Tasks;
+use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
+use Piwik\Tests\Fixtures\RawArchiveDataWithTempAndInvalidated;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * @group Core
+ */
+class TasksTest extends IntegrationTestCase
+{
+    /**
+     * @var RawArchiveDataWithTempAndInvalidated
+     */
+    public static $fixture;
+
+    /**
+     * @var Tasks
+     */
+    private $tasks;
+
+    /**
+     * @var Date
+     */
+    private $january;
+
+    /**
+     * @var Date
+     */
+    private $february;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        $this->january = Date::factory('2015-01-01');
+        $this->february = Date::factory('2015-02-01');
+
+        $archivePurger = new ArchivePurger();
+        $archivePurger->setTodayDate(Date::factory('2015-02-27'));
+        $archivePurger->setYesterdayDate(Date::factory('2015-02-26'));
+        $archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp());
+
+        $this->tasks = new Tasks($archivePurger);
+    }
+
+    public function test_purgeInvalidatedArchives_PurgesCorrectInvalidatedArchives_AndOnlyPurgesDataForDatesAndSites_InInvalidatedReportsDistributedList()
+    {
+        $this->setUpInvalidatedReportsDistributedList($dates = array($this->february));
+
+        $this->tasks->purgeInvalidatedArchives();
+
+        self::$fixture->assertInvalidatedArchivesPurged($this->february);
+        self::$fixture->assertInvalidatedArchivesNotPurged($this->january);
+
+        // assert invalidated reports distributed list has changed
+        $archivesToPurgeDistributedList = new ArchivesToPurgeDistributedList();
+        $yearMonths = $archivesToPurgeDistributedList->getAll();
+
+        $this->assertEmpty($yearMonths);
+    }
+
+    /**
+     * @param Date[] $dates
+     */
+    private function setUpInvalidatedReportsDistributedList($dates)
+    {
+        $yearMonths = array();
+        foreach ($dates as $date) {
+            $yearMonths[] = $date->toString('Y_m');
+        }
+
+        $archivesToPurgeDistributedList = new ArchivesToPurgeDistributedList();
+        $archivesToPurgeDistributedList->add($yearMonths);
+    }
+}
+
+TasksTest::$fixture = new RawArchiveDataWithTempAndInvalidated();
\ No newline at end of file
diff --git a/plugins/SitesManager/SitesManager.php b/plugins/SitesManager/SitesManager.php
index b9c5fa0736..93e1d8c2e8 100644
--- a/plugins/SitesManager/SitesManager.php
+++ b/plugins/SitesManager/SitesManager.php
@@ -9,8 +9,7 @@
 namespace Piwik\Plugins\SitesManager;
 
 use Piwik\Common;
-use Piwik\DataAccess\ArchiveInvalidator;
-use Piwik\Db;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Tracker\Cache;
 use Piwik\Tracker\Model as TrackerModel;
 
diff --git a/plugins/SitesManager/tests/Integration/SitesManagerTest.php b/plugins/SitesManager/tests/Integration/SitesManagerTest.php
index 135f505f01..1ba60ce972 100644
--- a/plugins/SitesManager/tests/Integration/SitesManagerTest.php
+++ b/plugins/SitesManager/tests/Integration/SitesManagerTest.php
@@ -10,7 +10,7 @@ namespace Piwik\Plugins\SitesManager\tests\Integration;
 
 use Piwik\Access;
 use Piwik\Cache;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Date;
 use Piwik\Plugins\SitesManager\SitesManager;
 use Piwik\Tests\Framework\Fixture;
diff --git a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php
new file mode 100644
index 0000000000..a6ef25a89f
--- /dev/null
+++ b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php
@@ -0,0 +1,396 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Tests\Fixtures;
+
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\ArchiveWriter;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Tests\Framework\Fixture;
+
+/**
+ * Fixture that inserts rows into archive tables for Jan. 2015 + Feb. 2015. The rows include
+ * done rows + metrics/blobs, and the done rows have values of DONE_OK_TEMPORARY,
+ * DONE_OK, DONE_INVALIDATED. There are also some custom range archives.
+ *
+ * This class is used to test archive purging.
+ */
+class RawArchiveDataWithTempAndInvalidated extends Fixture
+{
+    private static $dummyArchiveData = array(
+        // outdated temporary
+        array(
+            'idarchive' => 1,
+            'idsite' => 1,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+
+        array(
+            'idarchive' => 2,
+            'idsite' => 2,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-01',
+            'date2' => '2015-02-31',
+            'period' => 3,
+            'ts_archived' => '2015-02-18 10:10:10'
+        ),
+
+        array(
+            'idarchive' => 3,
+            'idsite' => 3,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-04',
+            'date2' => '2015-02-10',
+            'period' => 2,
+            'ts_archived' => '2015-02-10 12:34:56'
+        ),
+
+        array(
+            'idarchive' => 4,
+            'idsite' => 1,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-15',
+            'date2' => '2015-02-15',
+            'period' => 1,
+            'ts_archived' => '2015-02-15 08:12:13'
+        ),
+
+
+        // valid temporary
+        array( // only valid
+            'idarchive' => 5,
+            'idsite' => 1,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-27',
+            'date2' => '2015-02-27',
+            'period' => 1,
+            'ts_archived' => '2015-02-27 08:08:08'
+        ),
+
+        array(
+            'idarchive' => 6,
+            'idsite' => 2,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-26',
+            'date2' => '2015-02-26',
+            'period' => 1,
+            'ts_archived' => '2015-02-26 07:07:07'
+        ),
+
+        array(
+            'idarchive' => 7,
+            'idsite' => 3,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-01',
+            'date2' => '2015-02-28',
+            'period' => 3,
+            'ts_archived' => '2015-02-15 00:00:00'
+        ),
+
+        // custom ranges
+        array(
+            'idarchive' => 8,
+            'idsite' => 1,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-14',
+            'period' => 5,
+            'ts_archived' => '2015-02-27 00:00:00'
+        ),
+
+        array(
+            'idarchive' => 9,
+            'idsite' => 2,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-05',
+            'date2' => '2015-02-14',
+            'period' => 5,
+            'ts_archived' => '2015-02-15 00:00:00'
+        ),
+
+        array(
+            'idarchive' => 10,
+            'idsite' => 3,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-05',
+            'date2' => '2015-03-05',
+            'period' => 5,
+            'ts_archived' => '2015-02-26 00:00:00'
+        ),
+
+        // invalidated
+        array(
+            'idarchive' => 11,
+            'idsite' => 1,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_INVALIDATED,
+            'date1' => '2015-02-10',
+            'date2' => '2015-02-10',
+            'period' => 1,
+            'ts_archived' => '2015-02-10 12:13:14'
+        ),
+
+        array(
+            'idarchive' => 12,
+            'idsite' => 2,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_INVALIDATED,
+            'date1' => '2015-02-08',
+            'date2' => '2015-02-14',
+            'period' => 2,
+            'ts_archived' => '2015-02-15 00:00:00'
+        ),
+
+        array(
+            'idarchive' => 13,
+            'idsite' => 3,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_INVALIDATED,
+            'date1' => '2015-02-01',
+            'date2' => '2015-02-28',
+            'period' => 3,
+            'ts_archived' => '2015-02-27 13:13:13'
+        ),
+
+        array(
+            'idarchive' => 14,
+            'idsite' => 1,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_INVALIDATED,
+            'date1' => '2015-02-28',
+            'date2' => '2015-02-28',
+            'period' => 1,
+            'ts_archived' => '2015-02-28 12:12:12'
+        ),
+
+        array(
+            'idarchive' => 15,
+            'idsite' => 1,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_INVALIDATED,
+            'date1' => '2015-02-27',
+            'date2' => '2015-02-27',
+            'period' => 1,
+            'ts_archived' => '2015-02-28 12:12:12'
+        ),
+
+        // reprocessed invalidated
+        array(
+            'idarchive' => 16,
+            'idsite' => 1,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-10',
+            'date2' => '2015-02-10',
+            'period' => 1,
+            'ts_archived' => '2015-02-11 12:13:14'
+        ),
+
+        array(
+            'idarchive' => 17,
+            'idsite' => 2,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-08',
+            'date2' => '2015-02-14',
+            'period' => 2,
+            'ts_archived' => '2015-02-16 00:00:00'
+        ),
+
+        array(
+            'idarchive' => 18,
+            'idsite' => 3,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-01',
+            'date2' => '2015-02-28',
+            'period' => 3,
+            'ts_archived' => '2015-02-28 13:13:13'
+        ),
+
+        array(
+            'idarchive' => 19,
+            'idsite' => 1,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-28',
+            'date2' => '2015-02-28',
+            'period' => 1,
+            'ts_archived' => '2015-02-28 16:12:12' // must be late so it doesn't screw up the purgeOutdatedArchives test
+        ),
+    );
+
+    /**
+     * @var Date
+     */
+    public $january;
+
+    /**
+     * @var Date
+     */
+    public $february;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        $this->january = Date::factory('2015-01-01');
+        $this->february = Date::factory('2015-02-01');
+
+        $this->insertOutdatedArchives($this->january);
+        $this->insertOutdatedArchives($this->february);
+    }
+
+    private function insertOutdatedArchives(Date $archiveDate)
+    {
+        $dummyArchiveData = $this->getDummyArchiveDataForDate($archiveDate);
+
+        $numericTable = ArchiveTableCreator::getNumericTable($archiveDate);
+        foreach ($dummyArchiveData as $row) {
+            // done row
+            $this->insertTestArchiveRow($numericTable, $row);
+
+            // two metrics
+            $row['name'] = 'nb_visits';
+            $row['value'] = 1;
+            $this->insertTestArchiveRow($numericTable, $row);
+
+            $row['name'] = 'nb_actions';
+            $row['value'] = 2;
+            $this->insertTestArchiveRow($numericTable, $row);
+        }
+
+        $blobTable = ArchiveTableCreator::getBlobTable($archiveDate);
+        foreach ($dummyArchiveData as $row) {
+            // two blobs
+            $row['name'] = 'blobname';
+            $row['value'] = 'dummyvalue';
+            $this->insertTestArchiveRow($blobTable, $row);
+
+            $row['name'] = 'blobname2';
+            $row['value'] = 'dummyvalue';
+            $this->insertTestArchiveRow($blobTable, $row);
+        }
+    }
+
+    private function insertTestArchiveRow($table, $row)
+    {
+        $insertSqlTemplate = "INSERT INTO %s (idarchive, idsite, name, value, date1, date2, period, ts_archived) VALUES ('%s')";
+
+        Db::exec(sprintf($insertSqlTemplate, $table, implode("','", $row)));
+    }
+
+    private function getDummyArchiveDataForDate($archiveDate)
+    {
+        $rows = self::$dummyArchiveData;
+        foreach ($rows as &$row) {
+            $row['date1'] = $this->setDateMonthAndYear($row['date1'], $archiveDate);
+            $row['date2'] = $this->setDateMonthAndYear($row['date1'], $archiveDate);
+        }
+        return$rows;
+    }
+
+    private function setDateMonthAndYear($dateString, Date $archiveDate)
+    {
+        return $archiveDate->toString('Y-m') . '-' . Date::factory($dateString)->toString('d');
+    }
+
+    public function assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled, Date $date)
+    {
+        if ($isBrowserTriggeredArchivingEnabled) {
+            $expectedPurgedArchives = array(1,2,3,4,6,7); // only archives from 2 hours before "now" are purged
+        } else {
+            $expectedPurgedArchives = array(1,2,3,4,7); // only archives before start of "yesterday" are purged
+        }
+
+        $this->assertArchivesDoNotExist($expectedPurgedArchives, $date);
+    }
+
+    public function assertCustomRangesPurged(Date $date)
+    {
+        $expectedPurgedArchives = array(8,9,10);
+        $this->assertArchivesDoNotExist($expectedPurgedArchives, $date);
+    }
+
+    public function assertTemporaryArchivesNotPurged(Date $date)
+    {
+        $expectedPresentArchives = array(1,2,3,4,5,6,7);
+        $this->assertArchivesExist($expectedPresentArchives, $date);
+    }
+
+    public function assertInvalidatedArchivesNotPurged(Date $date)
+    {
+        $expectedPresentArchives = array(11, 12, 13, 14);
+        $this->assertArchivesExist($expectedPresentArchives, $date);
+    }
+
+    public function assertCustomRangesNotPurged(Date $date, $includeTemporary = true)
+    {
+        $expectedPresentArchives = array(8, 9);
+        if ($includeTemporary) {
+            $expectedPresentArchives[] = 10;
+        }
+        $this->assertArchivesExist($expectedPresentArchives, $date);
+    }
+
+    public function assertArchivesDoNotExist($expectedPurgedArchiveIds, $archiveDate)
+    {
+        $numericTable = ArchiveTableCreator::getNumericTable($archiveDate);
+        $blobTable = ArchiveTableCreator::getBlobTable($archiveDate);
+
+        $numericPurgedArchiveCount = $this->getArchiveRowCountWithId($numericTable, $expectedPurgedArchiveIds);
+        $this->assertEquals(0, $numericPurgedArchiveCount);
+
+        $blobPurgedArchiveCount = $this->getArchiveRowCountWithId($blobTable, $expectedPurgedArchiveIds);
+        $this->assertEquals(0, $blobPurgedArchiveCount);
+    }
+
+    public function assertArchivesExist($expectedPresentArchiveIds, $archiveDate)
+    {
+        $numericTable = ArchiveTableCreator::getNumericTable($archiveDate);
+        $blobTable = ArchiveTableCreator::getBlobTable($archiveDate);
+
+        $numericArchiveCount = $this->getArchiveRowCountWithId($numericTable, $expectedPresentArchiveIds);
+        $expectedNumericRowCount = count($expectedPresentArchiveIds) * 3; // two metrics + 1 done row
+        $this->assertEquals($expectedNumericRowCount, $numericArchiveCount);
+
+        $blobArchiveCount = $this->getArchiveRowCountWithId($blobTable, $expectedPresentArchiveIds);
+        $expectedBlobRowCount = count($expectedPresentArchiveIds) * 2; // two blob rows
+        $this->assertEquals($expectedBlobRowCount, $blobArchiveCount);
+    }
+
+    private function getArchiveRowCountWithId($table, $archiveIds)
+    {
+        return Db::fetchOne("SELECT COUNT(*) FROM $table WHERE idarchive IN (".implode(',', $archiveIds).")");
+    }
+
+    public function assertInvalidatedArchivesPurged(Date $date)
+    {
+        // check invalidated archives for all sites are purged
+        $expectedPurgedArchives = array(11, 12, 13, 14);
+        $this->assertArchivesDoNotExist($expectedPurgedArchives, $date);
+
+        // check archive 15 is not purged since it doesn't have newer DONE_OK/DONE_TEMPORARY archive
+        $expectedExistingArchives = array(15);
+        $this->assertArchivesExist($expectedExistingArchives, $date);
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php b/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php
index f88a62b445..3a8d33c7a5 100644
--- a/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php
+++ b/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php
@@ -8,8 +8,8 @@
 
 namespace Piwik\Tests\Framework\TestCase;
 
-use Piwik\Config;
 use Piwik\Console;
+use Symfony\Component\Console\Application;
 use Symfony\Component\Console\Tester\ApplicationTester;
 
 /**
@@ -37,18 +37,24 @@ use Symfony\Component\Console\Tester\ApplicationTester;
  */
 class ConsoleCommandTestCase extends SystemTestCase
 {
+    /**
+     * @var ApplicationTester
+     */
     protected $applicationTester = null;
 
+    /**
+     * @var Console
+     */
+    protected $application;
+
     public function setUp()
     {
         parent::setUp();
 
-        $application = new Console();
-        $application->setAutoExit(false);
-
-        $this->applicationTester = new ApplicationTester($application);
+        $this->application = new Console();
+        $this->application->setAutoExit(false);
 
-        Config::unsetInstance();
+        $this->applicationTester = new ApplicationTester($this->application);
     }
 
     protected function getCommandDisplayOutputErrorMessage()
diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php
new file mode 100644
index 0000000000..a422473479
--- /dev/null
+++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php
@@ -0,0 +1,117 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link    http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Tests\Integration\Archive;
+
+use Piwik\Archive\ArchivePurger;
+use Piwik\Config;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Tests\Fixtures\RawArchiveDataWithTempAndInvalidated;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * @group Core
+ */
+class PurgerTest extends IntegrationTestCase
+{
+    /**
+     * @var RawArchiveDataWithTempAndInvalidated
+     */
+    public static $fixture;
+
+    /**
+     * @var ArchivePurger
+     */
+    private $archivePurger;
+
+    /**
+     * @var Date
+     */
+    private $january;
+
+    /**
+     * @var Date
+     */
+    private $february;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        $this->january = self::$fixture->january;
+        $this->february = self::$fixture->february;
+
+        $this->archivePurger = new ArchivePurger();
+        $this->archivePurger->setTodayDate(Date::factory('2015-02-27'));
+        $this->archivePurger->setYesterdayDate(Date::factory('2015-02-26'));
+        $this->archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp());
+
+        $this->configureCustomRangePurging();
+
+        // assert test data was added correctly
+        self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->january);
+        self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->february);
+    }
+
+    public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringEnabled()
+    {
+        $this->enableBrowserTriggeredArchiving();
+
+        $this->archivePurger->purgeOutdatedArchives($this->february);
+
+        self::$fixture->assertTemporaryArchivesPurged($browserTriggeringEnabled = true, $this->february);
+
+        self::$fixture->assertCustomRangesNotPurged($this->february, $includeTemporary = false);
+        self::$fixture->assertTemporaryArchivesNotPurged($this->january);
+    }
+
+    public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringDisabled()
+    {
+        $this->disableBrowserTriggeredArchiving();
+
+        $this->archivePurger->purgeOutdatedArchives($this->february);
+
+        self::$fixture->assertTemporaryArchivesPurged($browserTriggeringEnabled = false, $this->february);
+
+        self::$fixture->assertCustomRangesNotPurged($this->february);
+        self::$fixture->assertTemporaryArchivesNotPurged($this->january);
+    }
+
+    public function test_purgeInvalidatedArchivesFrom_PurgesAllInvalidatedArchives_AndMarksDatesAndSitesAsInvalidated()
+    {
+        $this->archivePurger->purgeInvalidatedArchivesFrom($this->february);
+
+        self::$fixture->assertInvalidatedArchivesPurged($this->february);
+        self::$fixture->assertInvalidatedArchivesNotPurged($this->january);
+    }
+
+    public function test_purgeArchivesWithPeriodRange_PurgesAllRangeArchives()
+    {
+        $this->archivePurger->purgeArchivesWithPeriodRange($this->february);
+
+        self::$fixture->assertCustomRangesPurged($this->february);
+        self::$fixture->assertCustomRangesNotPurged($this->january);
+    }
+
+    private function configureCustomRangePurging()
+    {
+        Config::getInstance()->General['purge_date_range_archives_after_X_days'] = 3;
+    }
+
+    private function enableBrowserTriggeredArchiving()
+    {
+        Config::getInstance()->General['enable_browser_archiving_triggering'] = 1;
+    }
+
+    private function disableBrowserTriggeredArchiving()
+    {
+        Config::getInstance()->General['enable_browser_archiving_triggering'] = 0;
+    }
+}
+
+PurgerTest::$fixture = new RawArchiveDataWithTempAndInvalidated();
\ No newline at end of file
diff --git a/tests/PHPUnit/Integration/Concurrency/DistributedListTest.php b/tests/PHPUnit/Integration/Concurrency/DistributedListTest.php
new file mode 100644
index 0000000000..0facc27a74
--- /dev/null
+++ b/tests/PHPUnit/Integration/Concurrency/DistributedListTest.php
@@ -0,0 +1,153 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Tests\Integration\Concurrency;
+
+use Piwik\Common;
+use Piwik\Concurrency\DistributedList;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Option;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * @group Core
+ */
+class DistributedListTest extends IntegrationTestCase
+{
+    const TEST_OPTION_NAME = 'test.distributed.list';
+
+    public static $defaultOptionValues = array(
+        'val1',
+        'val2',
+        'val3',
+        'val4'
+    );
+
+    /**
+     * @var DistributedList
+     */
+    private $distributedList;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        $this->distributedList = new DistributedList(self::TEST_OPTION_NAME);
+
+        $this->initOptionValue();
+    }
+
+    public function test_getAll_CorrectlyReturnsItemsInOption()
+    {
+        $list = $this->distributedList->getAll();
+        $this->assertEquals(self::$defaultOptionValues, $list);
+    }
+
+    public function test_getAll_ReturnsValueInOption_IfOptionCacheHasSeparateValue()
+    {
+        // get option so cache is loaded
+        Option::get(self::TEST_OPTION_NAME);
+
+        // set option value to something else
+        $newList = array('1', '2', '3');
+        $this->initOptionValue($newList);
+
+        // test option is now different
+        $list = $this->distributedList->getAll();
+        $this->assertEquals($newList, $list);
+    }
+
+    public function test_setAll_CorrectlySetsNormalListInOption()
+    {
+        $newList = array('1', '2', '3');
+        $this->distributedList->setAll($newList);
+
+        $optionValue = $this->getOptionValueForList();
+        $this->assertEquals(serialize($newList), $optionValue);
+
+        $list = $this->distributedList->getAll();
+        $this->assertEquals($newList, $list);
+    }
+
+    public function test_setAll_CorrectlyConvertsItemsToString_BeforePersistingToOption()
+    {
+        $newList = array('1', Date::factory('2015-02-03'), 4.5);
+        $this->distributedList->setAll($newList);
+
+        $optionValue = $this->getOptionValueForList();
+        $expectedOptionList = array('1', '2015-02-03', '4.5');
+        $this->assertEquals(serialize($expectedOptionList), $optionValue);
+
+        $list = $this->distributedList->getAll();
+        $this->assertEquals($expectedOptionList, $list);
+    }
+
+    public function test_add_AddsOneItemToList_InOptionTable_IfItemIsNotArray()
+    {
+        $this->distributedList->add('val5');
+
+        $expectedOptionList = array('val1', 'val2', 'val3', 'val4', 'val5');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    public function test_add_AddsMultipleItemsToList_InOptionTable_IfItemsIsArray()
+    {
+        $this->distributedList->add(array('val5', Date::factory('2015-03-04')));
+
+        $expectedOptionList = array('val1', 'val2', 'val3', 'val4', 'val5', '2015-03-04');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    public function test_remove_RemovesSingleItemByValue_InOptionTable_IfItemIsNotArray()
+    {
+        $this->distributedList->remove('val2');
+
+        $expectedOptionList = array('val1', 'val3', 'val4');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    public function test_remove_RemovesMultipleItemsByValue_InOptionTable_IfItemIsArray()
+    {
+        $this->distributedList->remove(array('val2', 'val4'));
+
+        $expectedOptionList = array('val1', 'val3');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    public function test_removeByIndex_RemovesSingleItemByIndex_InOptionTable_IfArgIsIndex()
+    {
+        $this->distributedList->removeByIndex(2);
+
+        $expectedOptionList = array('val1', 'val2', 'val4');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    public function test_removeByIndex_RemovesMultipleItemsByIndex_InOptionTable_IfArgIsArray()
+    {
+        $this->distributedList->removeByIndex(array(1, 3));
+
+        $expectedOptionList = array('val1', 'val3');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    private function initOptionValue($data = false)
+    {
+        $data = $data ?: self::$defaultOptionValues;
+
+        $optionTable = Common::prefixTable('option');
+        Db::query("INSERT INTO `$optionTable` (option_name, option_value, autoload) VALUES (?, ?, ?)
+                   ON DUPLICATE KEY UPDATE option_value = ?",
+            array(self::TEST_OPTION_NAME, serialize($data), 0, serialize($data)));
+    }
+
+    private function getOptionValueForList()
+    {
+        $optionTable = Common::prefixTable('option');
+        return Db::fetchOne("SELECT option_value FROM `$optionTable` WHERE option_name = ?", array(self::TEST_OPTION_NAME));
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php
index a85f227802..c90e64a846 100644
--- a/tests/PHPUnit/Integration/CronArchiveTest.php
+++ b/tests/PHPUnit/Integration/CronArchiveTest.php
@@ -9,7 +9,7 @@
 namespace Piwik\Tests\Integration;
 
 use Piwik\CronArchive;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Plugins\CoreAdminHome\tests\Framework\Mock\API;
diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
index fbf2c0b7a9..7847abc202 100644
--- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
@@ -11,7 +11,7 @@ namespace Piwik\Tests\Integration\DataAccess;
 use Piwik\Date;
 use Piwik\Option;
 use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 
 /**
  * @group Archiver
diff --git a/tests/PHPUnit/Integration/Tracker/VisitTest.php b/tests/PHPUnit/Integration/Tracker/VisitTest.php
index bbbb972cc3..919a918516 100644
--- a/tests/PHPUnit/Integration/Tracker/VisitTest.php
+++ b/tests/PHPUnit/Integration/Tracker/VisitTest.php
@@ -11,7 +11,7 @@ namespace Piwik\Tests\Integration\Tracker;
 use Piwik\Access;
 use Piwik\Cache;
 use Piwik\CacheId;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Date;
 use Piwik\Network\IPUtils;
 use Piwik\Plugin\Manager;
diff --git a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php
index 56dc98e9a0..7e9a576b71 100755
--- a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php
+++ b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php
@@ -9,7 +9,7 @@ namespace Piwik\Tests\System;
 
 use Piwik\Archive;
 use Piwik\Cache;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Option;
 use Piwik\Plugins\Goals\Archiver;
 use Piwik\Segment;
diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
index 4f159e8399..77bb4f8dac 100755
--- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
+++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
@@ -8,8 +8,8 @@
 namespace Piwik\Tests\System;
 
 use Piwik\Common;
-use Piwik\DataAccess\ArchiveInvalidator;
-use Piwik\DataAccess\InvalidatedReports;
+use Piwik\Archive\ArchiveInvalidator;
+use Piwik\CronArchive\SitesToReprocessDistributedList;
 use Piwik\Db;
 use Piwik\Tests\Framework\TestCase\SystemTestCase;
 use Piwik\Tests\Fixtures\TwoVisitsWithCustomVariables;