Faster archiving of aggregated reports, also performance imprvovements in general

* Store subtables in chunks of 100 subtables per blob. Those 100 subtables are stored
  serialized as an array: array($subtableID => subtableBlob). The first 100 subtables are
  stored in "chunk_0", the next 100 subtables are stored in "chunk_1", ...
* Subtable Ids are now consecutive from 1 to X
* We do no longer serialize the whole Row instance when archiving, instead we only
  serialize the Row's array which contains columns, metadata and datatable. This is not
  only more efficient but allows us to refactor the Row instance in the future (although
  we will always have to be BC)
* Faster row implementation: Columns, Metadata and Subtables access is much faster now

1 files changed, 51 insertions, 4 deletions

diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
index 77bb4f8dac..6c75fb32d2 100755
--- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
+++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php
@@ -7,10 +7,12 @@
  */
 namespace Piwik\Tests\System;
 
+use Piwik\Archive\Chunk;
 use Piwik\Common;
 use Piwik\Archive\ArchiveInvalidator;
 use Piwik\CronArchive\SitesToReprocessDistributedList;
 use Piwik\Db;
+use Piwik\Piwik;
 use Piwik\Tests\Framework\TestCase\SystemTestCase;
 use Piwik\Tests\Fixtures\TwoVisitsWithCustomVariables;
 
@@ -74,9 +76,9 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest
             // 1) CHECK 'day' archive stored in January
             // We expect 2 segments
             //   * (1 custom variable name + 2 ref metrics
-            //      + 6 subtable for the custom var values + 5 Referrers blob
+            //      + 1 subtable for the custom var values + 5 Referrers blob (2 of them subtables)
             //   )
-            'archive_blob_2010_01'    => 28,
+            'archive_blob_2010_01'    => 18,
             // This contains all 'last N' weeks & days,
             // (2 metrics
             //  + 2 referrer metrics
@@ -86,8 +88,8 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest
             'archive_numeric_2010_01' => 138,
 
             // 2) CHECK 'week' archive stored in December (week starts the month before)
-            // We expect 2 segments * (1 custom variable name + 2 ref metrics + 5 subtable for the values of the name + 5 referrers blob)
-            'archive_blob_2009_12'    => 28,
+            // We expect 2 segments * (1 custom variable name + 2 ref metrics + 1 subtable for the values of the name + 5 referrers blob (2 of them subtables))
+            'archive_blob_2009_12'    => 18,
             // 7 metrics,
             // 2 Referrer metrics (Referrers_distinctSearchEngines/Referrers_distinctKeywords),
             // 6 done flag (referrers, CustomVar, VisitsSummary), 3 for period = 1 and 3 for period = 2
@@ -107,6 +109,51 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest
         }
     }
 
+    /**
+     *  Check that it merges all subtables into one blob entry
+     *
+     * @depends      testApi
+     */
+    public function test_checkArchiveRecords_shouldMergeSubtablesIntoOneRow()
+    {
+        $chunk = new Chunk();
+        $chunkBlobId = $chunk->getBlobIdForTable(0);
+
+        $tests = array(
+            'archive_blob_2010_01' => array(
+                'CustomVariables_valueByName_' . $chunkBlobId => 6,
+                'Referrers_keywordBySearchEngine_' . $chunkBlobId => 1,
+                'Referrers_searchEngineByKeyword_' . $chunkBlobId => 1,
+            ),
+            'archive_blob_2009_12' => array(
+                'CustomVariables_valueByName_' . $chunkBlobId => 6,
+                'Referrers_keywordBySearchEngine_' . $chunkBlobId => 1,
+                'Referrers_searchEngineByKeyword_' . $chunkBlobId => 1,
+            )
+        );
+        $numTests = 0;
+        foreach ($tests as $table => $expectedSubtables) {
+            foreach ($expectedSubtables as $name => $expectedNumSubtables) {
+                $sql = "SELECT `value` FROM " . Common::prefixTable($table) . " WHERE `name` ='$name'";
+                $blobs = Db::get()->fetchAll($sql);
+
+                foreach ($blobs as $blob) {
+                    $numTests++;
+                    $blob = $blob['value'];
+                    $blob = gzuncompress($blob);
+                    $blob = unserialize($blob);
+
+                    $countSubtables = count($blob);
+
+                    $this->assertEquals($expectedNumSubtables, $countSubtables, "$name in $table expected to contain $expectedNumSubtables subtables, got $countSubtables");
+                }
+            }
+        }
+
+        // 6 _subtables entries + 6 _subtables entries for the segment
+        $this->assertEquals(12, $numTests, "$numTests were executed but expected 12");
+    }
+
     public static function getOutputPrefix()
     {
         return 'twoVisitsWithCustomVariables_segmentMatchVisitorType';