Merge branch 'master' of github.com:piwik/piwik

18 files changed, 873 insertions, 14 deletions

diff --git a/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php
new file mode 100644
index 0000000000..a6ef25a89f
--- /dev/null
+++ b/tests/PHPUnit/Fixtures/RawArchiveDataWithTempAndInvalidated.php
@@ -0,0 +1,396 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Tests\Fixtures;
+
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataAccess\ArchiveWriter;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Tests\Framework\Fixture;
+
+/**
+ * Fixture that inserts rows into archive tables for Jan. 2015 + Feb. 2015. The rows include
+ * done rows + metrics/blobs, and the done rows have values of DONE_OK_TEMPORARY,
+ * DONE_OK, DONE_INVALIDATED. There are also some custom range archives.
+ *
+ * This class is used to test archive purging.
+ */
+class RawArchiveDataWithTempAndInvalidated extends Fixture
+{
+    private static $dummyArchiveData = array(
+        // outdated temporary
+        array(
+            'idarchive' => 1,
+            'idsite' => 1,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-03',
+            'period' => 1,
+            'ts_archived' => '2015-02-03 12:12:12'
+        ),
+
+        array(
+            'idarchive' => 2,
+            'idsite' => 2,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-01',
+            'date2' => '2015-02-31',
+            'period' => 3,
+            'ts_archived' => '2015-02-18 10:10:10'
+        ),
+
+        array(
+            'idarchive' => 3,
+            'idsite' => 3,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-04',
+            'date2' => '2015-02-10',
+            'period' => 2,
+            'ts_archived' => '2015-02-10 12:34:56'
+        ),
+
+        array(
+            'idarchive' => 4,
+            'idsite' => 1,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-15',
+            'date2' => '2015-02-15',
+            'period' => 1,
+            'ts_archived' => '2015-02-15 08:12:13'
+        ),
+
+
+        // valid temporary
+        array( // only valid
+            'idarchive' => 5,
+            'idsite' => 1,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-27',
+            'date2' => '2015-02-27',
+            'period' => 1,
+            'ts_archived' => '2015-02-27 08:08:08'
+        ),
+
+        array(
+            'idarchive' => 6,
+            'idsite' => 2,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-26',
+            'date2' => '2015-02-26',
+            'period' => 1,
+            'ts_archived' => '2015-02-26 07:07:07'
+        ),
+
+        array(
+            'idarchive' => 7,
+            'idsite' => 3,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-01',
+            'date2' => '2015-02-28',
+            'period' => 3,
+            'ts_archived' => '2015-02-15 00:00:00'
+        ),
+
+        // custom ranges
+        array(
+            'idarchive' => 8,
+            'idsite' => 1,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-03',
+            'date2' => '2015-02-14',
+            'period' => 5,
+            'ts_archived' => '2015-02-27 00:00:00'
+        ),
+
+        array(
+            'idarchive' => 9,
+            'idsite' => 2,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-05',
+            'date2' => '2015-02-14',
+            'period' => 5,
+            'ts_archived' => '2015-02-15 00:00:00'
+        ),
+
+        array(
+            'idarchive' => 10,
+            'idsite' => 3,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-05',
+            'date2' => '2015-03-05',
+            'period' => 5,
+            'ts_archived' => '2015-02-26 00:00:00'
+        ),
+
+        // invalidated
+        array(
+            'idarchive' => 11,
+            'idsite' => 1,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_INVALIDATED,
+            'date1' => '2015-02-10',
+            'date2' => '2015-02-10',
+            'period' => 1,
+            'ts_archived' => '2015-02-10 12:13:14'
+        ),
+
+        array(
+            'idarchive' => 12,
+            'idsite' => 2,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_INVALIDATED,
+            'date1' => '2015-02-08',
+            'date2' => '2015-02-14',
+            'period' => 2,
+            'ts_archived' => '2015-02-15 00:00:00'
+        ),
+
+        array(
+            'idarchive' => 13,
+            'idsite' => 3,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_INVALIDATED,
+            'date1' => '2015-02-01',
+            'date2' => '2015-02-28',
+            'period' => 3,
+            'ts_archived' => '2015-02-27 13:13:13'
+        ),
+
+        array(
+            'idarchive' => 14,
+            'idsite' => 1,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_INVALIDATED,
+            'date1' => '2015-02-28',
+            'date2' => '2015-02-28',
+            'period' => 1,
+            'ts_archived' => '2015-02-28 12:12:12'
+        ),
+
+        array(
+            'idarchive' => 15,
+            'idsite' => 1,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_INVALIDATED,
+            'date1' => '2015-02-27',
+            'date2' => '2015-02-27',
+            'period' => 1,
+            'ts_archived' => '2015-02-28 12:12:12'
+        ),
+
+        // reprocessed invalidated
+        array(
+            'idarchive' => 16,
+            'idsite' => 1,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-10',
+            'date2' => '2015-02-10',
+            'period' => 1,
+            'ts_archived' => '2015-02-11 12:13:14'
+        ),
+
+        array(
+            'idarchive' => 17,
+            'idsite' => 2,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-08',
+            'date2' => '2015-02-14',
+            'period' => 2,
+            'ts_archived' => '2015-02-16 00:00:00'
+        ),
+
+        array(
+            'idarchive' => 18,
+            'idsite' => 3,
+            'name' => 'done',
+            'value' => ArchiveWriter::DONE_OK,
+            'date1' => '2015-02-01',
+            'date2' => '2015-02-28',
+            'period' => 3,
+            'ts_archived' => '2015-02-28 13:13:13'
+        ),
+
+        array(
+            'idarchive' => 19,
+            'idsite' => 1,
+            'name' => 'doneDUMMYHASHSTR',
+            'value' => ArchiveWriter::DONE_OK_TEMPORARY,
+            'date1' => '2015-02-28',
+            'date2' => '2015-02-28',
+            'period' => 1,
+            'ts_archived' => '2015-02-28 16:12:12' // must be late so it doesn't screw up the purgeOutdatedArchives test
+        ),
+    );
+
+    /**
+     * @var Date
+     */
+    public $january;
+
+    /**
+     * @var Date
+     */
+    public $february;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        $this->january = Date::factory('2015-01-01');
+        $this->february = Date::factory('2015-02-01');
+
+        $this->insertOutdatedArchives($this->january);
+        $this->insertOutdatedArchives($this->february);
+    }
+
+    private function insertOutdatedArchives(Date $archiveDate)
+    {
+        $dummyArchiveData = $this->getDummyArchiveDataForDate($archiveDate);
+
+        $numericTable = ArchiveTableCreator::getNumericTable($archiveDate);
+        foreach ($dummyArchiveData as $row) {
+            // done row
+            $this->insertTestArchiveRow($numericTable, $row);
+
+            // two metrics
+            $row['name'] = 'nb_visits';
+            $row['value'] = 1;
+            $this->insertTestArchiveRow($numericTable, $row);
+
+            $row['name'] = 'nb_actions';
+            $row['value'] = 2;
+            $this->insertTestArchiveRow($numericTable, $row);
+        }
+
+        $blobTable = ArchiveTableCreator::getBlobTable($archiveDate);
+        foreach ($dummyArchiveData as $row) {
+            // two blobs
+            $row['name'] = 'blobname';
+            $row['value'] = 'dummyvalue';
+            $this->insertTestArchiveRow($blobTable, $row);
+
+            $row['name'] = 'blobname2';
+            $row['value'] = 'dummyvalue';
+            $this->insertTestArchiveRow($blobTable, $row);
+        }
+    }
+
+    private function insertTestArchiveRow($table, $row)
+    {
+        $insertSqlTemplate = "INSERT INTO %s (idarchive, idsite, name, value, date1, date2, period, ts_archived) VALUES ('%s')";
+
+        Db::exec(sprintf($insertSqlTemplate, $table, implode("','", $row)));
+    }
+
+    private function getDummyArchiveDataForDate($archiveDate)
+    {
+        $rows = self::$dummyArchiveData;
+        foreach ($rows as &$row) {
+            $row['date1'] = $this->setDateMonthAndYear($row['date1'], $archiveDate);
+            $row['date2'] = $this->setDateMonthAndYear($row['date1'], $archiveDate);
+        }
+        return$rows;
+    }
+
+    private function setDateMonthAndYear($dateString, Date $archiveDate)
+    {
+        return $archiveDate->toString('Y-m') . '-' . Date::factory($dateString)->toString('d');
+    }
+
+    public function assertTemporaryArchivesPurged($isBrowserTriggeredArchivingEnabled, Date $date)
+    {
+        if ($isBrowserTriggeredArchivingEnabled) {
+            $expectedPurgedArchives = array(1,2,3,4,6,7); // only archives from 2 hours before "now" are purged
+        } else {
+            $expectedPurgedArchives = array(1,2,3,4,7); // only archives before start of "yesterday" are purged
+        }
+
+        $this->assertArchivesDoNotExist($expectedPurgedArchives, $date);
+    }
+
+    public function assertCustomRangesPurged(Date $date)
+    {
+        $expectedPurgedArchives = array(8,9,10);
+        $this->assertArchivesDoNotExist($expectedPurgedArchives, $date);
+    }
+
+    public function assertTemporaryArchivesNotPurged(Date $date)
+    {
+        $expectedPresentArchives = array(1,2,3,4,5,6,7);
+        $this->assertArchivesExist($expectedPresentArchives, $date);
+    }
+
+    public function assertInvalidatedArchivesNotPurged(Date $date)
+    {
+        $expectedPresentArchives = array(11, 12, 13, 14);
+        $this->assertArchivesExist($expectedPresentArchives, $date);
+    }
+
+    public function assertCustomRangesNotPurged(Date $date, $includeTemporary = true)
+    {
+        $expectedPresentArchives = array(8, 9);
+        if ($includeTemporary) {
+            $expectedPresentArchives[] = 10;
+        }
+        $this->assertArchivesExist($expectedPresentArchives, $date);
+    }
+
+    public function assertArchivesDoNotExist($expectedPurgedArchiveIds, $archiveDate)
+    {
+        $numericTable = ArchiveTableCreator::getNumericTable($archiveDate);
+        $blobTable = ArchiveTableCreator::getBlobTable($archiveDate);
+
+        $numericPurgedArchiveCount = $this->getArchiveRowCountWithId($numericTable, $expectedPurgedArchiveIds);
+        $this->assertEquals(0, $numericPurgedArchiveCount);
+
+        $blobPurgedArchiveCount = $this->getArchiveRowCountWithId($blobTable, $expectedPurgedArchiveIds);
+        $this->assertEquals(0, $blobPurgedArchiveCount);
+    }
+
+    public function assertArchivesExist($expectedPresentArchiveIds, $archiveDate)
+    {
+        $numericTable = ArchiveTableCreator::getNumericTable($archiveDate);
+        $blobTable = ArchiveTableCreator::getBlobTable($archiveDate);
+
+        $numericArchiveCount = $this->getArchiveRowCountWithId($numericTable, $expectedPresentArchiveIds);
+        $expectedNumericRowCount = count($expectedPresentArchiveIds) * 3; // two metrics + 1 done row
+        $this->assertEquals($expectedNumericRowCount, $numericArchiveCount);
+
+        $blobArchiveCount = $this->getArchiveRowCountWithId($blobTable, $expectedPresentArchiveIds);
+        $expectedBlobRowCount = count($expectedPresentArchiveIds) * 2; // two blob rows
+        $this->assertEquals($expectedBlobRowCount, $blobArchiveCount);
+    }
+
+    private function getArchiveRowCountWithId($table, $archiveIds)
+    {
+        return Db::fetchOne("SELECT COUNT(*) FROM $table WHERE idarchive IN (".implode(',', $archiveIds).")");
+    }
+
+    public function assertInvalidatedArchivesPurged(Date $date)
+    {
+        // check invalidated archives for all sites are purged
+        $expectedPurgedArchives = array(11, 12, 13, 14);
+        $this->assertArchivesDoNotExist($expectedPurgedArchives, $date);
+
+        // check archive 15 is not purged since it doesn't have newer DONE_OK/DONE_TEMPORARY archive
+        $expectedExistingArchives = array(15);
+        $this->assertArchivesExist($expectedExistingArchives, $date);
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php b/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php
index f88a62b445..3a8d33c7a5 100644
--- a/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php
+++ b/tests/PHPUnit/Framework/TestCase/ConsoleCommandTestCase.php
@@ -8,8 +8,8 @@
 
 namespace Piwik\Tests\Framework\TestCase;
 
-use Piwik\Config;
 use Piwik\Console;
+use Symfony\Component\Console\Application;
 use Symfony\Component\Console\Tester\ApplicationTester;
 
 /**
@@ -37,18 +37,24 @@ use Symfony\Component\Console\Tester\ApplicationTester;
  */
 class ConsoleCommandTestCase extends SystemTestCase
 {
+    /**
+     * @var ApplicationTester
+     */
     protected $applicationTester = null;
 
+    /**
+     * @var Console
+     */
+    protected $application;
+
     public function setUp()
     {
         parent::setUp();
 
-        $application = new Console();
-        $application->setAutoExit(false);
-
-        $this->applicationTester = new ApplicationTester($application);
+        $this->application = new Console();
+        $this->application->setAutoExit(false);
 
-        Config::unsetInstance();
+        $this->applicationTester = new ApplicationTester($this->application);
     }
 
     protected function getCommandDisplayOutputErrorMessage()
diff --git a/tests/PHPUnit/Framework/TestCase/UnitTestCase.php b/tests/PHPUnit/Framework/TestCase/UnitTestCase.php
index f08eaa1953..8e07c39cc7 100755
--- a/tests/PHPUnit/Framework/TestCase/UnitTestCase.php
+++ b/tests/PHPUnit/Framework/TestCase/UnitTestCase.php
@@ -18,7 +18,7 @@ use Piwik\Tests\Framework\Mock\File;
  */
 abstract class UnitTestCase extends \PHPUnit_Framework_TestCase
 {
-    public function setup()
+    public function setUp()
     {
         parent::setUp();
         File::reset();
diff --git a/tests/PHPUnit/Integration/Archive/PurgerTest.php b/tests/PHPUnit/Integration/Archive/PurgerTest.php
new file mode 100644
index 0000000000..a422473479
--- /dev/null
+++ b/tests/PHPUnit/Integration/Archive/PurgerTest.php
@@ -0,0 +1,117 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link    http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Tests\Integration\Archive;
+
+use Piwik\Archive\ArchivePurger;
+use Piwik\Config;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Tests\Fixtures\RawArchiveDataWithTempAndInvalidated;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * @group Core
+ */
+class PurgerTest extends IntegrationTestCase
+{
+    /**
+     * @var RawArchiveDataWithTempAndInvalidated
+     */
+    public static $fixture;
+
+    /**
+     * @var ArchivePurger
+     */
+    private $archivePurger;
+
+    /**
+     * @var Date
+     */
+    private $january;
+
+    /**
+     * @var Date
+     */
+    private $february;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        $this->january = self::$fixture->january;
+        $this->february = self::$fixture->february;
+
+        $this->archivePurger = new ArchivePurger();
+        $this->archivePurger->setTodayDate(Date::factory('2015-02-27'));
+        $this->archivePurger->setYesterdayDate(Date::factory('2015-02-26'));
+        $this->archivePurger->setNow(Date::factory('2015-02-27 08:00:00')->getTimestamp());
+
+        $this->configureCustomRangePurging();
+
+        // assert test data was added correctly
+        self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->january);
+        self::$fixture->assertInvalidatedArchivesNotPurged(self::$fixture->february);
+    }
+
+    public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringEnabled()
+    {
+        $this->enableBrowserTriggeredArchiving();
+
+        $this->archivePurger->purgeOutdatedArchives($this->february);
+
+        self::$fixture->assertTemporaryArchivesPurged($browserTriggeringEnabled = true, $this->february);
+
+        self::$fixture->assertCustomRangesNotPurged($this->february, $includeTemporary = false);
+        self::$fixture->assertTemporaryArchivesNotPurged($this->january);
+    }
+
+    public function test_purgeOutdatedArchives_PurgesCorrectTemporaryArchives_WhileKeepingNewerTemporaryArchives_WithBrowserTriggeringDisabled()
+    {
+        $this->disableBrowserTriggeredArchiving();
+
+        $this->archivePurger->purgeOutdatedArchives($this->february);
+
+        self::$fixture->assertTemporaryArchivesPurged($browserTriggeringEnabled = false, $this->february);
+
+        self::$fixture->assertCustomRangesNotPurged($this->february);
+        self::$fixture->assertTemporaryArchivesNotPurged($this->january);
+    }
+
+    public function test_purgeInvalidatedArchivesFrom_PurgesAllInvalidatedArchives_AndMarksDatesAndSitesAsInvalidated()
+    {
+        $this->archivePurger->purgeInvalidatedArchivesFrom($this->february);
+
+        self::$fixture->assertInvalidatedArchivesPurged($this->february);
+        self::$fixture->assertInvalidatedArchivesNotPurged($this->january);
+    }
+
+    public function test_purgeArchivesWithPeriodRange_PurgesAllRangeArchives()
+    {
+        $this->archivePurger->purgeArchivesWithPeriodRange($this->february);
+
+        self::$fixture->assertCustomRangesPurged($this->february);
+        self::$fixture->assertCustomRangesNotPurged($this->january);
+    }
+
+    private function configureCustomRangePurging()
+    {
+        Config::getInstance()->General['purge_date_range_archives_after_X_days'] = 3;
+    }
+
+    private function enableBrowserTriggeredArchiving()
+    {
+        Config::getInstance()->General['enable_browser_archiving_triggering'] = 1;
+    }
+
+    private function disableBrowserTriggeredArchiving()
+    {
+        Config::getInstance()->General['enable_browser_archiving_triggering'] = 0;
+    }
+}
+
+PurgerTest::$fixture = new RawArchiveDataWithTempAndInvalidated();
\ No newline at end of file
diff --git a/tests/PHPUnit/Integration/Concurrency/DistributedListTest.php b/tests/PHPUnit/Integration/Concurrency/DistributedListTest.php
new file mode 100644
index 0000000000..0facc27a74
--- /dev/null
+++ b/tests/PHPUnit/Integration/Concurrency/DistributedListTest.php
@@ -0,0 +1,153 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Tests\Integration\Concurrency;
+
+use Piwik\Common;
+use Piwik\Concurrency\DistributedList;
+use Piwik\Date;
+use Piwik\Db;
+use Piwik\Option;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * @group Core
+ */
+class DistributedListTest extends IntegrationTestCase
+{
+    const TEST_OPTION_NAME = 'test.distributed.list';
+
+    public static $defaultOptionValues = array(
+        'val1',
+        'val2',
+        'val3',
+        'val4'
+    );
+
+    /**
+     * @var DistributedList
+     */
+    private $distributedList;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        $this->distributedList = new DistributedList(self::TEST_OPTION_NAME);
+
+        $this->initOptionValue();
+    }
+
+    public function test_getAll_CorrectlyReturnsItemsInOption()
+    {
+        $list = $this->distributedList->getAll();
+        $this->assertEquals(self::$defaultOptionValues, $list);
+    }
+
+    public function test_getAll_ReturnsValueInOption_IfOptionCacheHasSeparateValue()
+    {
+        // get option so cache is loaded
+        Option::get(self::TEST_OPTION_NAME);
+
+        // set option value to something else
+        $newList = array('1', '2', '3');
+        $this->initOptionValue($newList);
+
+        // test option is now different
+        $list = $this->distributedList->getAll();
+        $this->assertEquals($newList, $list);
+    }
+
+    public function test_setAll_CorrectlySetsNormalListInOption()
+    {
+        $newList = array('1', '2', '3');
+        $this->distributedList->setAll($newList);
+
+        $optionValue = $this->getOptionValueForList();
+        $this->assertEquals(serialize($newList), $optionValue);
+
+        $list = $this->distributedList->getAll();
+        $this->assertEquals($newList, $list);
+    }
+
+    public function test_setAll_CorrectlyConvertsItemsToString_BeforePersistingToOption()
+    {
+        $newList = array('1', Date::factory('2015-02-03'), 4.5);
+        $this->distributedList->setAll($newList);
+
+        $optionValue = $this->getOptionValueForList();
+        $expectedOptionList = array('1', '2015-02-03', '4.5');
+        $this->assertEquals(serialize($expectedOptionList), $optionValue);
+
+        $list = $this->distributedList->getAll();
+        $this->assertEquals($expectedOptionList, $list);
+    }
+
+    public function test_add_AddsOneItemToList_InOptionTable_IfItemIsNotArray()
+    {
+        $this->distributedList->add('val5');
+
+        $expectedOptionList = array('val1', 'val2', 'val3', 'val4', 'val5');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    public function test_add_AddsMultipleItemsToList_InOptionTable_IfItemsIsArray()
+    {
+        $this->distributedList->add(array('val5', Date::factory('2015-03-04')));
+
+        $expectedOptionList = array('val1', 'val2', 'val3', 'val4', 'val5', '2015-03-04');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    public function test_remove_RemovesSingleItemByValue_InOptionTable_IfItemIsNotArray()
+    {
+        $this->distributedList->remove('val2');
+
+        $expectedOptionList = array('val1', 'val3', 'val4');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    public function test_remove_RemovesMultipleItemsByValue_InOptionTable_IfItemIsArray()
+    {
+        $this->distributedList->remove(array('val2', 'val4'));
+
+        $expectedOptionList = array('val1', 'val3');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    public function test_removeByIndex_RemovesSingleItemByIndex_InOptionTable_IfArgIsIndex()
+    {
+        $this->distributedList->removeByIndex(2);
+
+        $expectedOptionList = array('val1', 'val2', 'val4');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    public function test_removeByIndex_RemovesMultipleItemsByIndex_InOptionTable_IfArgIsArray()
+    {
+        $this->distributedList->removeByIndex(array(1, 3));
+
+        $expectedOptionList = array('val1', 'val3');
+        $this->assertEquals(serialize($expectedOptionList), $this->getOptionValueForList());
+    }
+
+    private function initOptionValue($data = false)
+    {
+        $data = $data ?: self::$defaultOptionValues;
+
+        $optionTable = Common::prefixTable('option');
+        Db::query("INSERT INTO `$optionTable` (option_name, option_value, autoload) VALUES (?, ?, ?)
+                   ON DUPLICATE KEY UPDATE option_value = ?",
+            array(self::TEST_OPTION_NAME, serialize($data), 0, serialize($data)));
+    }
+
+    private function getOptionValueForList()
+    {
+        $optionTable = Common::prefixTable('option');
+        return Db::fetchOne("SELECT option_value FROM `$optionTable` WHERE option_name = ?", array(self::TEST_OPTION_NAME));
+    }
+}
\ No newline at end of file
diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php
index a85f227802..c90e64a846 100644
--- a/tests/PHPUnit/Integration/CronArchiveTest.php
+++ b/tests/PHPUnit/Integration/CronArchiveTest.php
@@ -9,7 +9,7 @@
 namespace Piwik\Tests\Integration;
 
 use Piwik\CronArchive;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Date;
 use Piwik\Db;
 use Piwik\Plugins\CoreAdminHome\tests\Framework\Mock\API;
diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
index fbf2c0b7a9..7847abc202 100644
--- a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php
@@ -11,7 +11,7 @@ namespace Piwik\Tests\Integration\DataAccess;
 use Piwik\Date;
 use Piwik\Option;
 use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 
 /**
  * @group Archiver
diff --git a/tests/PHPUnit/Integration/Tracker/ModelTest.php b/tests/PHPUnit/Integration/Tracker/ModelTest.php
index def08a3316..2490c8a072 100644
--- a/tests/PHPUnit/Integration/Tracker/ModelTest.php
+++ b/tests/PHPUnit/Integration/Tracker/ModelTest.php
@@ -9,6 +9,7 @@ namespace Piwik\Tests\Integration\Tracker;
 
 use Piwik\Common;
 use Piwik\Db;
+use Piwik\Tests\Fixtures\OneVisitorTwoVisits;
 use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
 use Piwik\Tracker\Model;
 
@@ -111,6 +112,16 @@ class ModelTest extends IntegrationTestCase
         $this->assertEquals($expectedResult, $result);
     }
 
+    public function test_isSiteEmpty()
+    {
+        $this->assertTrue($this->model->isSiteEmpty(1));
+
+        $fixture = new OneVisitorTwoVisits();
+        $fixture->setUp();
+
+        $this->assertFalse($this->model->isSiteEmpty(1));
+    }
+
     private function assertLogActionTableContainsTestAction($idaction)
     {
         $expectedRows = array(
diff --git a/tests/PHPUnit/Integration/Tracker/VisitTest.php b/tests/PHPUnit/Integration/Tracker/VisitTest.php
index bbbb972cc3..919a918516 100644
--- a/tests/PHPUnit/Integration/Tracker/VisitTest.php
+++ b/tests/PHPUnit/Integration/Tracker/VisitTest.php
@@ -11,7 +11,7 @@ namespace Piwik\Tests\Integration\Tracker;
 use Piwik\Access;
 use Piwik\Cache;
 use Piwik\CacheId;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Date;
 use Piwik\Network\IPUtils;
 use Piwik\Plugin\Manager;
diff --git a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeTest.php b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeTest.php
index a0bdae8d8f..2e7051911f 100755
--- a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeTest.php
+++ b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeTest.php
@@ -71,6 +71,11 @@ class OneVisitorOneWebsiteSeveralDaysDateRangeTest extends SystemTestCase
             array('Referrers.getSocials', array('idSite'  => 'all',
                                                'date'    => '2010-12-13,2011-01-18',
                                                'periods' => array('range'))),
+            array('Referrers.getSocials', array('idSite'  => $idSite,
+                                               'date'     => '2010-12-13,2011-01-18',
+                                                'testSuffix' => '_Flattened',
+                                                'otherRequestParameters' => array('flat' => '1'),
+                                                'periods' => array('range'))),
 
             array('Referrers.getSocials', array('idSite'       => 'all',
                                                'date'         => '2010-12-10',
diff --git a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php
index 56dc98e9a0..7e9a576b71 100755
--- a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php
+++ b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php
@@ -9,7 +9,7 @@ namespace Piwik\Tests\System;
 
 use Piwik\Archive;
 use Piwik\Cache;
-use Piwik\DataAccess\ArchiveInvalidator;
+use Piwik\Archive\ArchiveInvalidator;
 use Piwik\Option;
 use Piwik\Plugins\Goals\Archiver;
 use Piwik\Segment;
diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
index 4f159e8399..77bb4f8dac 100755
--- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
+++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
@@ -8,8 +8,8 @@
 namespace Piwik\Tests\System;
 
 use Piwik\Common;
-use Piwik\DataAccess\ArchiveInvalidator;
-use Piwik\DataAccess\InvalidatedReports;
+use Piwik\Archive\ArchiveInvalidator;
+use Piwik\CronArchive\SitesToReprocessDistributedList;
 use Piwik\Db;
 use Piwik\Tests\Framework\TestCase\SystemTestCase;
 use Piwik\Tests\Fixtures\TwoVisitsWithCustomVariables;
diff --git a/tests/PHPUnit/System/expected/test_oneVisitor_oneWebsite_severalDays_DateRange_Flattened__Referrers.getSocials_range.xml b/tests/PHPUnit/System/expected/test_oneVisitor_oneWebsite_severalDays_DateRange_Flattened__Referrers.getSocials_range.xml
new file mode 100644
index 0000000000..7f335fe207
--- /dev/null
+++ b/tests/PHPUnit/System/expected/test_oneVisitor_oneWebsite_severalDays_DateRange_Flattened__Referrers.getSocials_range.xml
@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<result>
+	<row>
+		<label>Facebook - facebook.com/whatever</label>
+		<nb_visits>1</nb_visits>
+		<nb_actions>2</nb_actions>
+		<max_actions>2</max_actions>
+		<sum_visit_length>361</sum_visit_length>
+		<bounce_count>0</bounce_count>
+		<nb_visits_converted>0</nb_visits_converted>
+		<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+		<sum_daily_nb_users>0</sum_daily_nb_users>
+		<url>http://facebook.com/whatever</url>
+		<logo>plugins/Referrers/images/socials/facebook.com.png</logo>
+	</row>
+	<row>
+		<label>Facebook - fb.me/?q=sdlfjs&amp;n=slfjsd</label>
+		<nb_visits>1</nb_visits>
+		<nb_actions>2</nb_actions>
+		<max_actions>2</max_actions>
+		<sum_visit_length>361</sum_visit_length>
+		<bounce_count>0</bounce_count>
+		<nb_visits_converted>0</nb_visits_converted>
+		<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+		<sum_daily_nb_users>0</sum_daily_nb_users>
+		<url>http://fb.me/?q=sdlfjs&amp;n=slfjsd</url>
+		<logo>plugins/Referrers/images/socials/facebook.com.png</logo>
+	</row>
+	<row>
+		<label>Facebook - www.facebook.com/another/path</label>
+		<nb_visits>1</nb_visits>
+		<nb_actions>1</nb_actions>
+		<max_actions>1</max_actions>
+		<sum_visit_length>0</sum_visit_length>
+		<bounce_count>1</bounce_count>
+		<nb_visits_converted>0</nb_visits_converted>
+		<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+		<sum_daily_nb_users>0</sum_daily_nb_users>
+		<url>http://www.facebook.com/another/path</url>
+		<logo>plugins/Referrers/images/socials/facebook.com.png</logo>
+	</row>
+	<row>
+		<label>Mixi - mixi.jp</label>
+		<nb_visits>1</nb_visits>
+		<nb_actions>1</nb_actions>
+		<max_actions>1</max_actions>
+		<sum_visit_length>0</sum_visit_length>
+		<bounce_count>1</bounce_count>
+		<nb_visits_converted>0</nb_visits_converted>
+		<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+		<sum_daily_nb_users>0</sum_daily_nb_users>
+		<url>http://mixi.jp</url>
+		<logo>plugins/Referrers/images/socials/mixi.jp.png</logo>
+	</row>
+	<row>
+		<label>Skyrock - skyrock.com</label>
+		<nb_visits>1</nb_visits>
+		<nb_actions>2</nb_actions>
+		<max_actions>2</max_actions>
+		<sum_visit_length>361</sum_visit_length>
+		<bounce_count>0</bounce_count>
+		<nb_visits_converted>0</nb_visits_converted>
+		<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+		<sum_daily_nb_users>0</sum_daily_nb_users>
+		<url>http://skyrock.com</url>
+		<logo>plugins/Referrers/images/socials/skyrock.com.png</logo>
+	</row>
+	<row>
+		<label>Twitter - t.co/id/?y=dsfs</label>
+		<nb_visits>1</nb_visits>
+		<nb_actions>1</nb_actions>
+		<max_actions>1</max_actions>
+		<sum_visit_length>0</sum_visit_length>
+		<bounce_count>1</bounce_count>
+		<nb_visits_converted>0</nb_visits_converted>
+		<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+		<sum_daily_nb_users>0</sum_daily_nb_users>
+		<url>http://t.co/id/?y=dsfs</url>
+		<logo>plugins/Referrers/images/socials/twitter.com.png</logo>
+	</row>
+	<row>
+		<label>Twitter - twitter.com/whatever2</label>
+		<nb_visits>1</nb_visits>
+		<nb_actions>1</nb_actions>
+		<max_actions>1</max_actions>
+		<sum_visit_length>0</sum_visit_length>
+		<bounce_count>1</bounce_count>
+		<nb_visits_converted>0</nb_visits_converted>
+		<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+		<sum_daily_nb_users>0</sum_daily_nb_users>
+		<url>http://twitter.com/whatever2</url>
+		<logo>plugins/Referrers/images/socials/twitter.com.png</logo>
+	</row>
+	<row>
+		<label>Twitter - www.twitter.com/index?a=2334</label>
+		<nb_visits>1</nb_visits>
+		<nb_actions>2</nb_actions>
+		<max_actions>2</max_actions>
+		<sum_visit_length>361</sum_visit_length>
+		<bounce_count>0</bounce_count>
+		<nb_visits_converted>0</nb_visits_converted>
+		<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+		<sum_daily_nb_users>0</sum_daily_nb_users>
+		<url>http://www.twitter.com/index?a=2334</url>
+		<logo>plugins/Referrers/images/socials/twitter.com.png</logo>
+	</row>
+	<row>
+		<label>Flickr - www.flickr.com</label>
+		<nb_visits>1</nb_visits>
+		<nb_actions>2</nb_actions>
+		<max_actions>2</max_actions>
+		<sum_visit_length>361</sum_visit_length>
+		<bounce_count>0</bounce_count>
+		<nb_visits_converted>0</nb_visits_converted>
+		<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+		<sum_daily_nb_users>0</sum_daily_nb_users>
+		<url>http://www.flickr.com</url>
+		<logo>plugins/Referrers/images/socials/flickr.com.png</logo>
+	</row>
+	<row>
+		<label>Xanga - xanga.com</label>
+		<nb_visits>1</nb_visits>
+		<nb_actions>1</nb_actions>
+		<max_actions>1</max_actions>
+		<sum_visit_length>0</sum_visit_length>
+		<bounce_count>1</bounce_count>
+		<nb_visits_converted>0</nb_visits_converted>
+		<sum_daily_nb_uniq_visitors>1</sum_daily_nb_uniq_visitors>
+		<sum_daily_nb_users>0</sum_daily_nb_users>
+		<url>http://xanga.com</url>
+		<logo>plugins/Referrers/images/socials/xanga.com.png</logo>
+	</row>
+</result>
\ No newline at end of file
diff --git a/tests/PHPUnit/System/expected/test_reportLimiting_flattened__Referrers.getKeywords_day.xml b/tests/PHPUnit/System/expected/test_reportLimiting_flattened__Referrers.getKeywords_day.xml
index d093798ebf..30316b9941 100644
--- a/tests/PHPUnit/System/expected/test_reportLimiting_flattened__Referrers.getKeywords_day.xml
+++ b/tests/PHPUnit/System/expected/test_reportLimiting_flattened__Referrers.getKeywords_day.xml
@@ -10,6 +10,8 @@
 		<sum_visit_length>0</sum_visit_length>
 		<bounce_count>1</bounce_count>
 		<nb_visits_converted>0</nb_visits_converted>
+		<url>http://ask.com/web?q=this+search+term</url>
+		<logo>plugins/Referrers/images/searchEngines/ask.com.png</logo>
 	</row>
 	<row>
 		<label>this search term - Others</label>
@@ -21,6 +23,7 @@
 		<sum_visit_length>0</sum_visit_length>
 		<bounce_count>2</bounce_count>
 		<nb_visits_converted>0</nb_visits_converted>
+		<logo>plugins/Referrers/images/searchEngines/xx.png</logo>
 	</row>
 	<row>
 		<label>search term 2 - Alexa</label>
@@ -32,6 +35,8 @@
 		<sum_visit_length>0</sum_visit_length>
 		<bounce_count>1</bounce_count>
 		<nb_visits_converted>0</nb_visits_converted>
+		<url>http://alexa.com/search?q=search+term+2</url>
+		<logo>plugins/Referrers/images/searchEngines/alexa.com.png</logo>
 	</row>
 	<row>
 		<label>search term 2 - Babylon</label>
@@ -43,6 +48,8 @@
 		<sum_visit_length>0</sum_visit_length>
 		<bounce_count>1</bounce_count>
 		<nb_visits_converted>0</nb_visits_converted>
+		<url>http://search.babylon.com/?q=search+term+2</url>
+		<logo>plugins/Referrers/images/searchEngines/search.babylon.com.png</logo>
 	</row>
 	<row>
 		<label>Others</label>
diff --git a/tests/PHPUnit/System/expected/test_reportLimiting_flattened__Referrers.getSearchEngines_day.xml b/tests/PHPUnit/System/expected/test_reportLimiting_flattened__Referrers.getSearchEngines_day.xml
index cb1d70e35b..915b5ddb39 100644
--- a/tests/PHPUnit/System/expected/test_reportLimiting_flattened__Referrers.getSearchEngines_day.xml
+++ b/tests/PHPUnit/System/expected/test_reportLimiting_flattened__Referrers.getSearchEngines_day.xml
@@ -10,6 +10,7 @@
 		<sum_visit_length>0</sum_visit_length>
 		<bounce_count>1</bounce_count>
 		<nb_visits_converted>0</nb_visits_converted>
+		<url>http://google.com/search?q=search+term+3</url>
 		<logo>plugins/Referrers/images/searchEngines/google.com.png</logo>
 	</row>
 	<row>
@@ -34,6 +35,7 @@
 		<sum_visit_length>0</sum_visit_length>
 		<bounce_count>1</bounce_count>
 		<nb_visits_converted>0</nb_visits_converted>
+		<url>http://search.yahoo.com/search?p=search+term+4</url>
 		<logo>plugins/Referrers/images/searchEngines/search.yahoo.com.png</logo>
 	</row>
 	<row>
diff --git a/tests/UI/expected-ui-screenshots b/tests/UI/expected-ui-screenshots
-Subproject b0d54bbd96340acf137cc37b92d8290cdaedf3c
+Subproject 560c44710748f4f27ceaee0955f59aa1336ee4a
diff --git a/tests/UI/specs/EmptySite_spec.js b/tests/UI/specs/EmptySite_spec.js
new file mode 100644
index 0000000000..704bda8721
--- /dev/null
+++ b/tests/UI/specs/EmptySite_spec.js
@@ -0,0 +1,22 @@
+/*!
+ * Piwik - free/libre analytics platform
+ *
+ * Screenshot integration tests.
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+describe("EmptySite", function () {
+    this.timeout(0);
+
+    var generalParams = 'idSite=4&period=day&date=2010-01-03';
+
+    it('should show the tracking code if the website has no recorded data', function (done) {
+        var urlToTest = "?" + generalParams + "&module=CoreHome&action=index";
+
+        expect.screenshot('emptySiteDashboard').to.be.captureSelector('.site-without-data', function (page) {
+            page.load(urlToTest);
+        }, done);
+    });
+});
diff --git a/tests/javascript/index.php b/tests/javascript/index.php
index 20fade48ea..072dd42538 100644
--- a/tests/javascript/index.php
+++ b/tests/javascript/index.php
@@ -2370,7 +2370,7 @@ function PiwikTest() {
     }
 
     test("User ID and Visitor UUID", function() {
-        expect(16);
+        expect(19);
         deleteCookies();
 
         var userIdString = 'userid@mydomain.org';
@@ -2396,6 +2396,13 @@ function PiwikTest() {
         var visitorId = tracker.getVisitorId();
         equal(visitorId, tracker.getVisitorId(), "Visitor ID is the same when called multiple times");
 
+        // Check that setting an empty user id will not change the visitor ID
+        var userId = '';
+        equal(userId, tracker.getUserId(), "by default user ID is set to empty string");
+        tracker.setUserId(userId);
+        equal(userId, tracker.getUserId(), "after setting to empty string, user id is still empty");
+        equal(visitorId, tracker.getVisitorId(), "visitor id was not changed after setting empty user id");
+
         // Building another 'tracker2' object so we can compare behavior to 'tracker'
         var tracker2 = Piwik.getTracker();
         equal(tracker.getVisitorId(), tracker2.getVisitorId(), "Visitor ID " + tracker.getVisitorId() + " is the same as Visitor ID 2 " + tracker2.getVisitorId());