Faster archiving of aggregated reports, also performance imprvovements in general

* Store subtables in chunks of 100 subtables per blob. Those 100 subtables are stored
  serialized as an array: array($subtableID => subtableBlob). The first 100 subtables are
  stored in "chunk_0", the next 100 subtables are stored in "chunk_1", ...
* Subtable Ids are now consecutive from 1 to X
* We do no longer serialize the whole Row instance when archiving, instead we only
  serialize the Row's array which contains columns, metadata and datatable. This is not
  only more efficient but allows us to refactor the Row instance in the future (although
  we will always have to be BC)
* Faster row implementation: Columns, Metadata and Subtables access is much faster now

1 files changed, 47 insertions, 17 deletions

diff --git a/core/DataTable.php b/core/DataTable.php
index c0a4aa9113..a668591221 100644
--- a/core/DataTable.php
+++ b/core/DataTable.php
@@ -17,13 +17,13 @@ use Piwik\DataTable\Renderer\Html;
 use Piwik\DataTable\Row;
 use Piwik\DataTable\Row\DataTableSummaryRow;
 use Piwik\DataTable\Simple;
-use Piwik\DataTable\TableNotFoundException;
 use ReflectionClass;
 
 /**
  * @see Common::destroy()
  */
 require_once PIWIK_INCLUDE_PATH . '/core/Common.php';
+require_once PIWIK_INCLUDE_PATH . "/core/DataTable/Bridges.php";
 
 /**
  * The primary data structure used to store analytics data in Piwik.
@@ -335,15 +335,25 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess
             foreach ($this->rows as $row) {
                 Common::destroy($row);
             }
-            if (!is_null($this->summaryRow)) {
+            if (isset($this->summaryRow)) {
                 Common::destroy($this->summaryRow);
             }
             unset($this->rows);
-            Manager::getInstance()->setTableDeleted($this->getId());
+            Manager::getInstance()->setTableDeleted($this->currentId);
             $depth--;
         }
     }
 
+    /**
+     * Clone. Called when cloning the datatable. We need to make sure to create a new datatableId.
+     * If we do not increase tableId it can result in segmentation faults when destructing a datatable.
+     */
+    public function __clone()
+    {
+        // registers this instance to the manager
+        $this->currentId = Manager::getInstance()->addTable($this);
+    }
+
     public function setLabelsHaveChanged()
     {
         $this->indexNotUpToDate = true;
@@ -670,6 +680,7 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess
                 $this->rowsIndexByLabel[$label] = $id;
             }
         }
+
         if ($this->summaryRow) {
             $label = $this->summaryRow->getColumn('label');
             if ($label !== false) {
@@ -1200,9 +1211,12 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess
                                   $columnToSortByBeforeTruncation = null)
     {
         static $depth = 0;
+        // make sure subtableIds are consecutive from 1 to N
+        static $subtableId = 0;
 
         if ($depth > self::$maximumDepthLevelAllowed) {
             $depth = 0;
+            $subtableId = 0;
             throw new Exception("Maximum recursion level of " . self::$maximumDepthLevelAllowed . " reached. Maybe you have set a DataTable\Row with an associated DataTable belonging already to one of its parent tables?");
         }
         if (!is_null($maximumRowsInDataTable)) {
@@ -1214,13 +1228,17 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess
             );
         }
 
+        $consecutiveSubtableIds = array();
+        $forcedId = $subtableId;
+
         // For each row, get the serialized row
         // If it is associated to a sub table, get the serialized table recursively ;
         // but returns all serialized tables and subtable in an array of 1 dimension
         $aSerializedDataTable = array();
-        foreach ($this->rows as $row) {
+        foreach ($this->rows as $id => $row) {
             $subTable = $row->getSubtable();
             if ($subTable) {
+                $consecutiveSubtableIds[$id] = ++$subtableId;
                 $depth++;
                 $aSerializedDataTable = $aSerializedDataTable + $subTable->getSerialized($maximumRowsInSubDataTable, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation);
                 $depth--;
@@ -1228,22 +1246,33 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess
                 $row->removeSubtable();
             }
         }
-        // we load the current Id of the DataTable
-        $forcedId = $this->getId();
 
         // if the datatable is the parent we force the Id at 0 (this is part of the specification)
         if ($depth == 0) {
             $forcedId = 0;
+            $subtableId = 0;
         }
 
         // we then serialize the rows and store them in the serialized dataTable
-        $addToRows = array(self::ID_SUMMARY_ROW => $this->summaryRow);
+        $rows = array();
+        foreach ($this->rows as $id => $row) {
+            if (array_key_exists($id, $consecutiveSubtableIds)) {
+                $backup = $row->subtableId;
+                $row->subtableId = $consecutiveSubtableIds[$id];
+                $rows[$id] = $row->export();
+                $row->subtableId = $backup;
+            } else {
+                $rows[$id] = $row->export();
+            }
+        }
 
-        $aSerializedDataTable[$forcedId] = serialize($this->rows + $addToRows);
-        foreach ($this->rows as &$row) {
-            $row->cleanPostSerialize();
+        if (isset($this->summaryRow)) {
+            $rows[self::ID_SUMMARY_ROW] = $this->summaryRow->export();
         }
 
+        $aSerializedDataTable[$forcedId] = serialize($rows);
+        unset($rows);
+
         return $aSerializedDataTable;
     }
 
@@ -1254,19 +1283,19 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess
      *
      * _Note: This function will successfully load DataTables serialized by Piwik 1.X._
      *
-     * @param string $stringSerialized A string with the format of a string in the array returned by
+     * @param string $serialized A string with the format of a string in the array returned by
      *                                 {@link serialize()}.
-     * @throws Exception if `$stringSerialized` is invalid.
+     * @throws Exception if `$serialized` is invalid.
      */
-    public function addRowsFromSerializedArray($stringSerialized)
+    public function addRowsFromSerializedArray($serialized)
     {
-        require_once PIWIK_INCLUDE_PATH . "/core/DataTable/Bridges.php";
+        $rows = unserialize($serialized);
 
-        $serialized = unserialize($stringSerialized);
-        if ($serialized === false) {
+        if ($rows === false) {
             throw new Exception("The unserialization has failed!");
         }
-        $this->addRowsFromArray($serialized);
+
+        $this->addRowsFromArray($rows);
     }
 
     /**
@@ -1292,6 +1321,7 @@ class DataTable implements DataTableInterface, \IteratorAggregate, \ArrayAccess
             if (is_array($row)) {
                 $row = new Row($row);
             }
+
             if ($id == self::ID_SUMMARY_ROW) {
                 $this->summaryRow = $row;
             } else {