A chunk implementation that is much simpler and makes more sense. Everything is now in the ArchiveWriter + Selector

6 files changed, 88 insertions, 231 deletions

diff --git a/tests/PHPUnit/Integration/ArchiveTest.php b/tests/PHPUnit/Integration/ArchiveTest.php
index f217fdaca7..906cfe40c9 100644
--- a/tests/PHPUnit/Integration/ArchiveTest.php
+++ b/tests/PHPUnit/Integration/ArchiveTest.php
@@ -196,18 +196,18 @@ class ArchiveTest extends IntegrationTestCase
      */
     public function test_ArchiveBlob_ShouldBeFindBlobs_WithinDifferentChunks($idSubtable, $expectedBlob)
     {
+        $recordName = 'Actions_Actions';
+
         $chunk    = new Chunk();
-        $chunk5   = $chunk->getBlobIdForTable($subtableId = 5);
-        $chunk152 = $chunk->getBlobIdForTable($subtableId = 152);
-        $chunk399 = $chunk->getBlobIdForTable($subtableId = 399);
+        $chunk5   = $chunk->getRecordNameForTableId($recordName, $subtableId = 5);
+        $chunk152 = $chunk->getRecordNameForTableId($recordName, $subtableId = 152);
+        $chunk399 = $chunk->getRecordNameForTableId($recordName, $subtableId = 399);
 
         $this->createArchiveBlobEntry('2013-01-02', array(
-            'Actions_Actions' => array(
-                0 => 'actions_02',
-                $chunk5   => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')),
-                $chunk152 => serialize(array(151 => 'actionsSubtable151', 152 => 'actionsSubtable152')),
-                $chunk399 => serialize(array(399 => 'actionsSubtable399')),
-            )
+            $recordName => 'actions_02',
+            $chunk5     => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')),
+            $chunk152   => serialize(array(151 => 'actionsSubtable151', 152 => 'actionsSubtable152')),
+            $chunk399   => serialize(array(399 => 'actionsSubtable399'))
         ));
 
         $archive = $this->getArchive('day', '2013-01-02,2013-01-02');
@@ -231,49 +231,39 @@ class ArchiveTest extends IntegrationTestCase
 
     private function createManyDifferentArchiveBlobs()
     {
+        $recordName1 = 'Actions_Actions';
+        $recordName2 = 'Actions_Actionsurl';
+
         $chunk = new Chunk();
-        $chunk0 = $chunk->getBlobIdForTable(0);
+        $chunk0_1 = $chunk->getRecordNameForTableId($recordName1, 0);
+        $chunk0_2 = $chunk->getRecordNameForTableId($recordName2, 0);
 
         $this->createArchiveBlobEntry('2013-01-01', array(
-            'Actions_Actionsurl' => array(
-                0 => 'test01'
-            )
+            $recordName2 => 'test01'
         ));
         $this->createArchiveBlobEntry('2013-01-02', array(
-            'Actions_Actionsurl' => array(
-                0 => 'test02',
-                1 => 'test1',
-                2 => 'test2'
-            ),
-            'Actions_Actions' => array(
-                0 => 'actions_02',
-                $chunk0 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5'))
-            )
+            $recordName2        => 'test02',
+            $recordName2 . '_1' => 'test1', // testing BC where each subtable was stored seperately
+            $recordName2 . '_2' => 'test2', // testing BC
+            $recordName1        => 'actions_02',
+            $chunk0_1 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5'))
         ));
         $this->createArchiveBlobEntry('2013-01-03', array(
-            'Actions_Actionsurl' => array(
-                0 => 'test03',
-                $chunk0 => serialize(array(1 => 'subtable1', 2 => 'subtable2', 5 => 'subtable5'))
-            ),
-            'Actions_Actions' => array(
-                0 => 'actions_03',
-                1 => 'actionsTest1',
-                2 => 'actionsTest2')
-            )
-        );
+            $recordName2 => 'test03',
+            $chunk0_2 => serialize(array(1 => 'subtable1', 2 => 'subtable2', 5 => 'subtable5')),
+            $recordName1 => 'actions_03',
+            $recordName1 . '_1' => 'actionsTest1',
+            $recordName1 . '_2' => 'actionsTest2'
+        ));
         $this->createArchiveBlobEntry('2013-01-04', array(
-            'Actions_Actionsurl' => array(
-                0 => 'test04',
-                5 => 'subtable45',
-                6 => 'subtable6')
-            )
-        );
+            $recordName2 => 'test04',
+            $recordName2 . '_5' => 'subtable45',
+            $recordName2 . '_6' => 'subtable6'
+        ));
         $this->createArchiveBlobEntry('2013-01-06', array(
-            'Actions_Actionsurl' => array(
-                0 => 'test06',
-                $chunk0 => serialize(array()))
-            )
-        );
+            $recordName2 => 'test06',
+            $chunk0_2 => serialize(array())
+        ));
     }
 
     private function assertArchiveBlob(PiwikArchive\DataCollection $dataCollection, $date, $expectedBlob)
diff --git a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTestsTest.php b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTestsTest.php
index 87e0a8294f..d879c07fb7 100755
--- a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTestsTest.php
+++ b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTestsTest.php
@@ -183,7 +183,7 @@ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCa
         );
         $chunk = new Chunk();
         foreach ($tests as $table => $expectedNumSubtables) {
-            $chunkAppendix = $chunk->getBlobIdForTable(0);
+            $chunkAppendix = $chunk->getRecordNameForTableId(0);
             $sql = "SELECT value FROM " . Common::prefixTable($table) . " WHERE period = " . Piwik::$idPeriods['range'] . " and `name` ='Actions_actions_url_$chunkAppendix'";
             $blob = Db::get()->fetchOne($sql);
             $blob = gzuncompress($blob);
diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
index 6c75fb32d2..a2c1d06672 100755
--- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
+++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
@@ -117,7 +117,7 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest
     public function test_checkArchiveRecords_shouldMergeSubtablesIntoOneRow()
     {
         $chunk = new Chunk();
-        $chunkBlobId = $chunk->getBlobIdForTable(0);
+        $chunkBlobId = $chunk->getRecordNameForTableId(0);
 
         $tests = array(
             'archive_blob_2010_01' => array(
diff --git a/tests/PHPUnit/Unit/Archive/ChunkTest.php b/tests/PHPUnit/Unit/Archive/ChunkTest.php
index 29904e7439..dcab94c267 100644
--- a/tests/PHPUnit/Unit/Archive/ChunkTest.php
+++ b/tests/PHPUnit/Unit/Archive/ChunkTest.php
@@ -24,6 +24,8 @@ class ChunkTest extends UnitTestCase
      */
     private $chunk;
 
+    private $recordName = 'Actions_ActionsUrl';
+
     public function setUp()
     {
         parent::setUp();
@@ -31,14 +33,14 @@ class ChunkTest extends UnitTestCase
     }
 
     /**
-     * @dataProvider getBlobIdForTableDataProvider
+     * @dataProvider getRecordNameForTableIdDataProvider
      */
-    public function test_getBlobIdForTable_shouldSplitChunksIntoBitsOf100($expectedChunk, $tableId)
+    public function test_getRecordNameForTableId_shouldSplitChunksIntoBitsOf100($expectedChunk, $tableId)
     {
-        $this->assertEquals('chunk_' . $expectedChunk, $this->chunk->getBlobIdForTable($tableId));
+        $this->assertEquals($this->recordName . '_chunk_' . $expectedChunk, $this->chunk->getRecordNameForTableId($this->recordName, $tableId));
     }
 
-    public function getBlobIdForTableDataProvider()
+    public function getRecordNameForTableId()
     {
         return array(
             array($expectedChunk = '0_99', $tableId = 0),
@@ -57,31 +59,6 @@ class ChunkTest extends UnitTestCase
     }
 
     /**
-     * @dataProvider isBlobIdAChunkDataProvider
-     */
-    public function test_isBlobIdAChunk($isChunk, $blobId)
-    {
-        $this->assertSame($isChunk, $this->chunk->isBlobIdAChunk($blobId));
-    }
-
-    public function isBlobIdAChunkDataProvider()
-    {
-        return array(
-            array($isChunk = true, $blobId = 'chunk_0_99'),
-            array(true, 'chunk_100_199'),
-            // following 2 are not really a chunk but we accept it as such for simpler/faster implementation
-            array(true, 'chunk_0'),
-            array(true, 'chunk_999'),
-            array(false, 'chunk0'),
-            array(false, 'chunk999'),
-            array(false, '0'),
-            array(false, '5'),
-            array(false, 5),
-            array(false, '_5'),
-        );
-    }
-
-    /**
      * @dataProvider isRecordNameAChunkDataProvider
      */
     public function test_isRecordNameAChunk_shouldSplitChunksIntoBitsOf100($isChunk, $recordName)
@@ -109,7 +86,7 @@ class ChunkTest extends UnitTestCase
 
     public function test_moveArchiveBlobsIntoChunks_NoChunksGiven()
     {
-        $this->assertSame(array(), $this->chunk->moveArchiveBlobsIntoChunks(array()));
+        $this->assertSame(array(), $this->chunk->moveArchiveBlobsIntoChunks($this->recordName, array()));
     }
 
     /**
@@ -119,12 +96,12 @@ class ChunkTest extends UnitTestCase
     {
         $array = array_fill(0, 245, 'test');
         $expected = array(
-            'chunk_0_99' => array_fill(0, Chunk::NUM_TABLES_IN_CHUNK, 'test'),
-            'chunk_100_199' => array_fill(100, Chunk::NUM_TABLES_IN_CHUNK, 'test'),
-            'chunk_200_299' => array_fill(200, 45, 'test'),
+            $this->recordName . '_chunk_0_99' => array_fill(0, Chunk::NUM_TABLES_IN_CHUNK, 'test'),
+            $this->recordName . '_chunk_100_199' => array_fill(100, Chunk::NUM_TABLES_IN_CHUNK, 'test'),
+            $this->recordName . '_chunk_200_299' => array_fill(200, 45, 'test'),
         );
 
-        $this->assertSame($expected, $this->chunk->moveArchiveBlobsIntoChunks($array));
+        $this->assertSame($expected, $this->chunk->moveArchiveBlobsIntoChunks($this->recordName, $array));
     }
 
     /**
diff --git a/tests/PHPUnit/Unit/ArchiveProcessorTest.php b/tests/PHPUnit/Unit/ArchiveProcessorTest.php
deleted file mode 100644
index 1f092e3d03..0000000000
--- a/tests/PHPUnit/Unit/ArchiveProcessorTest.php
+++ /dev/null
@@ -1,129 +0,0 @@
-<?php
-/**
- * Piwik - free/libre analytics platform
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- */
-
-namespace Piwik\Tests\Unit;
-
-use Piwik\Archive\Chunk;
-use Piwik\ArchiveProcessor;
-use Piwik\ArchiveProcessor\Parameters;
-use Piwik\DataTable;
-use Piwik\Segment;
-use Piwik\Tests\Framework\Mock\Site;
-use Piwik\Tests\Framework\TestCase\UnitTestCase;
-use Piwik\Period\Factory as PeriodFactory;
-
-/**
- * @group ArchiveProcessorTest
- * @group ArchiveProcessor
- * @group Archive
- * @group Core
- */
-class ArchiveProcessorTest extends UnitTestCase
-{
-
-    public function test_insertBlobRecord_NoBlobsGiven()
-    {
-        $this->assertInsertBlobRecordPassesBlobsToArchiveWriter(array(), array());
-    }
-
-    public function test_insertBlobRecord_OnlyRootTableGiven_ShouldNotMoveRootTableIntoAChunk()
-    {
-        $blobs = array(0 => $this->getSerializedBlob());
-        $this->assertInsertBlobRecordPassesBlobsToArchiveWriter($blobs, $blobs);
-    }
-
-    public function test_insertBlobRecord_RootAndSubTablesGiven_OnlyAfewSubtables()
-    {
-        $blobs = $this->generateBlobs(0, 45);
-
-        $expectedBlobs = array(
-            0         => $this->getSerializedBlob('_0'),
-            'chunk_0_99' => serialize($this->generateBlobs(1, 44)), // does not start with zero as zero is root table
-        );
-
-        $this->assertInsertBlobRecordPassesBlobsToArchiveWriter($expectedBlobs, $blobs);
-    }
-
-    public function test_insertBlobRecord_RootAndSubTablesGiven_ShouldOnlySplitSubtablesIntoAChunk()
-    {
-        $blobs = $this->generateBlobs(0, 1145);
-
-        $expectedBlobs = array(
-            0 => $this->getSerializedBlob('_0'),
-            'chunk_0_99'      => serialize($this->generateBlobs(1, Chunk::NUM_TABLES_IN_CHUNK - 1)), // does not start with zero as zero is root table
-            'chunk_100_199'   => serialize($this->generateBlobs(100, Chunk::NUM_TABLES_IN_CHUNK)),
-            'chunk_200_299'   => serialize($this->generateBlobs(200, Chunk::NUM_TABLES_IN_CHUNK)),
-            'chunk_300_399'   => serialize($this->generateBlobs(300, Chunk::NUM_TABLES_IN_CHUNK)),
-            'chunk_400_499'   => serialize($this->generateBlobs(400, Chunk::NUM_TABLES_IN_CHUNK)),
-            'chunk_500_599'   => serialize($this->generateBlobs(500, Chunk::NUM_TABLES_IN_CHUNK)),
-            'chunk_600_699'   => serialize($this->generateBlobs(600, Chunk::NUM_TABLES_IN_CHUNK)),
-            'chunk_700_799'   => serialize($this->generateBlobs(700, Chunk::NUM_TABLES_IN_CHUNK)),
-            'chunk_800_899'   => serialize($this->generateBlobs(800, Chunk::NUM_TABLES_IN_CHUNK)),
-            'chunk_900_999'   => serialize($this->generateBlobs(900, Chunk::NUM_TABLES_IN_CHUNK)),
-            'chunk_1000_1099' => serialize($this->generateBlobs(1000, Chunk::NUM_TABLES_IN_CHUNK)),
-            'chunk_1100_1199' => serialize($this->generateBlobs(1100, 45)),
-        );
-
-        $this->assertInsertBlobRecordPassesBlobsToArchiveWriter($expectedBlobs, $blobs);
-    }
-
-    public function test_insertBlobRecord_ShouldBeAbleToHandleAString()
-    {
-        $serialized = $this->getSerializedBlob();
-
-        $this->assertInsertBlobRecordPassesBlobsToArchiveWriter($serialized, $serialized);
-    }
-
-    private function generateBlobs($startIndex, $numberOfEntries)
-    {
-        $blobs = array();
-
-        for ($i = 0; $i < $numberOfEntries; $i++) {
-            $subtableId = $startIndex + $i;
-            // we need to append something to make sure it actually moves the correct blob into the correct chunk
-            $blobs[$subtableId] = $this->getSerializedBlob('_'. $subtableId);
-        }
-
-        return $blobs;
-    }
-
-    private function getSerializedBlob($appendix = '')
-    {
-        return 'a:1:{i:0;a:3:{i:0;a:0:{}i:1;a:0:{}i:3;N;}}' . $appendix;
-    }
-
-    private function assertInsertBlobRecordPassesBlobsToArchiveWriter($expectedBlobs, $blobs)
-    {
-        $recordName = 'Actions_Action_url';
-
-        $writer = $this->getMock('Piwik\DataAccess\ArchiveWriter', array('insertBlobRecord'), array(), '', false);
-        $writer->expects($this->once())
-            ->method('insertBlobRecord')
-            ->with($recordName, $expectedBlobs);
-
-        $processor = $this->createProcessor($writer);
-        $processor->insertBlobRecord($recordName, $blobs);
-    }
-
-    private function createArchiveProcessorParamaters()
-    {
-        $oPeriod = PeriodFactory::makePeriodFromQueryParams('UTC', 'day', '2015-01-01');
-
-        $segment = new Segment(false, array(1));
-        $params  = new Parameters(new Site(1), $oPeriod, $segment);
-
-        return $params;
-    }
-
-    private function createProcessor($writer)
-    {
-        $params  = $this->createArchiveProcessorParamaters();
-
-        return new ArchiveProcessor($params, $writer);
-    }
-}
\ No newline at end of file
diff --git a/tests/PHPUnit/Unit/DataAccess/ArchiveWriterTest.php b/tests/PHPUnit/Unit/DataAccess/ArchiveWriterTest.php
index edc77e1ba9..292dd7682a 100644
--- a/tests/PHPUnit/Unit/DataAccess/ArchiveWriterTest.php
+++ b/tests/PHPUnit/Unit/DataAccess/ArchiveWriterTest.php
@@ -8,6 +8,7 @@
 
 namespace Piwik\Tests\Unit;
 
+use Piwik\Archive\Chunk;
 use Piwik\DataAccess\ArchiveWriter;
 use Piwik\DataTable;
 use Piwik\Segment;
@@ -30,39 +31,44 @@ class ArchiveWriterTest extends UnitTestCase
         $this->assertInsertBlobRecordInsertedRecordsInBulk(array(), array());
     }
 
-    public function test_insertBlobRecord_ShouldAppendTheRecordNameToSubtables()
+    public function test_insertBlobRecord_OnlyRootTableGiven_ShouldNotMoveRootTableIntoAChunk()
     {
-        $blobs = array(
-            0 => $this->getSerializedBlob('_root'),
-            1 => $this->getSerializedBlob('subtable1'),
-            4 => $this->getSerializedBlob('subtable4'),
-            5 => $this->getSerializedBlob('subtable5')
-        );
+        $blobs    = array(0 => $this->getSerializedBlob());
+        $expected = array(array($this->recordName, $this->getSerializedBlob()));
+
+        $this->assertInsertBlobRecordInsertedRecordsInBulk($expected, $blobs);
+    }
+
+    public function test_insertBlobRecord_RootAndSubTablesGiven_OnlyAfewSubtables()
+    {
+        $blobs = $this->generateBlobs(0, 45);
 
         $expectedBlobs = array(
-            array($this->recordName       , $this->getSerializedBlob('_root')),
-            array($this->recordName . '_1', $this->getSerializedBlob('subtable1')),
-            array($this->recordName . '_4', $this->getSerializedBlob('subtable4')),
-            array($this->recordName . '_5', $this->getSerializedBlob('subtable5'))
+            array($this->recordName, $this->getSerializedBlob('_0')),
+            array($this->recordName . '_chunk_0_99', serialize($this->generateBlobs(1, 44)))
         );
 
         $this->assertInsertBlobRecordInsertedRecordsInBulk($expectedBlobs, $blobs);
     }
 
-    public function test_insertBlobRecord_ShouldAppendTheRecordNameToChunks()
+    public function test_insertBlobRecord_RootAndSubTablesGiven_ShouldOnlySplitSubtablesIntoAChunk()
     {
-        $blobs = array(
-            0 => $this->getSerializedBlob('_root'),
-            'chunk_0_99' => $this->getSerializedBlob('chunk0'),
-            'chunk_100_199' => $this->getSerializedBlob('chunk1'),
-            'chunk_200_299' => $this->getSerializedBlob('chunk2')
-        );
+        $blobs = $this->generateBlobs(0, 1145);
 
         $expectedBlobs = array(
-            array($this->recordName             , $this->getSerializedBlob('_root')),
-            array($this->recordName . '_chunk_0_99', $this->getSerializedBlob('chunk0')),
-            array($this->recordName . '_chunk_100_199', $this->getSerializedBlob('chunk1')),
-            array($this->recordName . '_chunk_200_299', $this->getSerializedBlob('chunk2'))
+            array($this->recordName, $this->getSerializedBlob('_0')),
+            array($this->recordName . '_chunk_0_99'     , serialize($this->generateBlobs(1, Chunk::NUM_TABLES_IN_CHUNK - 1))), // does not start with zero as zero is root table
+            array($this->recordName . '_chunk_100_199'  , serialize($this->generateBlobs(100, Chunk::NUM_TABLES_IN_CHUNK))),
+            array($this->recordName . '_chunk_200_299'  , serialize($this->generateBlobs(200, Chunk::NUM_TABLES_IN_CHUNK))),
+            array($this->recordName . '_chunk_300_399'  , serialize($this->generateBlobs(300, Chunk::NUM_TABLES_IN_CHUNK))),
+            array($this->recordName . '_chunk_400_499'  , serialize($this->generateBlobs(400, Chunk::NUM_TABLES_IN_CHUNK))),
+            array($this->recordName . '_chunk_500_599'  , serialize($this->generateBlobs(500, Chunk::NUM_TABLES_IN_CHUNK))),
+            array($this->recordName . '_chunk_600_699'  , serialize($this->generateBlobs(600, Chunk::NUM_TABLES_IN_CHUNK))),
+            array($this->recordName . '_chunk_700_799'  , serialize($this->generateBlobs(700, Chunk::NUM_TABLES_IN_CHUNK))),
+            array($this->recordName . '_chunk_800_899'  , serialize($this->generateBlobs(800, Chunk::NUM_TABLES_IN_CHUNK))),
+            array($this->recordName . '_chunk_900_999'  , serialize($this->generateBlobs(900, Chunk::NUM_TABLES_IN_CHUNK))),
+            array($this->recordName . '_chunk_1000_1099', serialize($this->generateBlobs(1000, Chunk::NUM_TABLES_IN_CHUNK))),
+            array($this->recordName . '_chunk_1100_1199', serialize($this->generateBlobs(1100, 45))),
         );
 
         $this->assertInsertBlobRecordInsertedRecordsInBulk($expectedBlobs, $blobs);
@@ -75,6 +81,19 @@ class ArchiveWriterTest extends UnitTestCase
         $this->assertInsertBlobRecordInsertedASingleRecord($blob, $blob);
     }
 
+    private function generateBlobs($startIndex, $numberOfEntries)
+    {
+        $blobs = array();
+
+        for ($i = 0; $i < $numberOfEntries; $i++) {
+            $subtableId = $startIndex + $i;
+            // we need to append something to make sure it actually moves the correct blob into the correct chunk
+            $blobs[$subtableId] = $this->getSerializedBlob('_'. $subtableId);
+        }
+
+        return $blobs;
+    }
+
     private function getSerializedBlob($appendix = '')
     {
         return 'a:1:{i:0;a:3:{i:0;a:0:{}i:1;a:0:{}i:3;N;}}' . $appendix;
@@ -83,7 +102,7 @@ class ArchiveWriterTest extends UnitTestCase
     private function assertInsertBlobRecordInsertedRecordsInBulk($expectedBlobs, $blobs)
     {
         $writer = $this->getMock('Piwik\DataAccess\ArchiveWriter', array('insertBulkRecords', 'compress'), array(), '', false);
-        $writer->expects($this->exactly(count($blobs)))
+        $writer->expects($this->exactly(count($expectedBlobs)))
                ->method('compress')
                ->will($this->returnArgument(0));
         $writer->expects($this->once())