Refactored archive data querying code. Removed tree-like data structure from Piwik_Archive, added memory optimizations for archiving (not storing archive processing instances), added optimization for querying data indexed by site + date, added ability to select multiple blobs/metrics at once (from different plugins), added optimization to Piwik_ArchiveProcessing_Period where numerics/records are selected all at once instead of one at a time.

Also added data access layer object for archive querying code, allowed anonymous function objects to be used as DataTable filters, and removed 'timestamp' DataTable metadata.

5 files changed, 19 insertions, 10 deletions

diff --git a/plugins/Goals/API.php b/plugins/Goals/API.php
index aea16ad70e..ad58ec4310 100644
--- a/plugins/Goals/API.php
+++ b/plugins/Goals/API.php
@@ -210,6 +210,7 @@ class Piwik_Goals_API
         }
         $archive = Piwik_Archive::build($idSite, $period, $date);
         $dataTable = $archive->getDataTable($recordNameFinal);
+        
         $dataTable->filter('Sort', array(Piwik_Archive::INDEX_ECOMMERCE_ITEM_REVENUE));
         $dataTable->queueFilter('ReplaceColumnNames');
         $dataTable->queueFilter('ReplaceSummaryRowLabel');
@@ -428,7 +429,7 @@ class Piwik_Goals_API
     {
         Piwik::checkUserHasViewAccess($idSite);
         $archive = Piwik_Archive::build($idSite, $period, $date, $segment);
-        $dataTable = $archive->getNumeric($toFetch);
+        $dataTable = $archive->getDataTableFromNumeric($toFetch);
         return $dataTable;
     }
 
diff --git a/plugins/MultiSites/API.php b/plugins/MultiSites/API.php
index b6b0a4d1ae..5aba1c4764 100755
--- a/plugins/MultiSites/API.php
+++ b/plugins/MultiSites/API.php
@@ -218,7 +218,9 @@ class Piwik_MultiSites_API
         ) {
             $dataTable = $dataTable->mergeChildren();
         } else {
-            if (!$dataTable instanceof Piwik_DataTable_Array) {
+            if (!$dataTable instanceof Piwik_DataTable_Array
+                && $dataTable->getRowsCount() > 0
+            ) {
                 $firstDataTableRow = $dataTable->getFirstRow();
                 $firstDataTableRow->setColumn('label', $sites);
             }
diff --git a/plugins/UserSettings/API.php b/plugins/UserSettings/API.php
index e438b96b2f..dd3820fff7 100644
--- a/plugins/UserSettings/API.php
+++ b/plugins/UserSettings/API.php
@@ -171,7 +171,7 @@ class Piwik_UserSettings_API
         $dataTable = $this->getDataTable('UserSettings_plugin', $idSite, $period, $date, $segment);
         $browserTypes = $this->getDataTable('UserSettings_browserType', $idSite, $period, $date, $segment);
         $archive = Piwik_Archive::build($idSite, $period, $date, $segment);
-        $visitsSums = $archive->getNumeric('nb_visits');
+        $visitsSums = $archive->getDataTableFromNumeric('nb_visits');
 
         // check whether given tables are arrays
         if ($dataTable instanceof Piwik_DataTable_Array) {
@@ -179,9 +179,9 @@ class Piwik_UserSettings_API
             $browserTypesArray = $browserTypes->getArray();
             $visitSumsArray = $visitsSums->getArray();
         } else {
-            $tableArray = Array($dataTable);
-            $browserTypesArray = Array($browserTypes);
-            $visitSumsArray = Array($visitsSums);
+            $tableArray = array($dataTable);
+            $browserTypesArray = array($browserTypes);
+            $visitSumsArray = array($visitsSums);
         }
 
         // walk through the results and calculate the percentage
@@ -198,7 +198,11 @@ class Piwik_UserSettings_API
             foreach ($visitSumsArray AS $k => $visits) {
                 if ($k == $key) {
                     if (is_object($visits)) {
-                        $visitsSumTotal = (float)$visits->getFirstRow()->getColumn(0);
+                        if ($visits->getRowsCount() == 0) {
+                            $visitsSumTotal = 0;
+                        } else {
+                            $visitsSumTotal = (float)$visits->getFirstRow()->getColumn('nb_visits');
+                        }
                     } else {
                         $visitsSumTotal = (float)$visits;
                     }
diff --git a/plugins/VisitTime/API.php b/plugins/VisitTime/API.php
index 3da0646fb7..973a8d3dd8 100644
--- a/plugins/VisitTime/API.php
+++ b/plugins/VisitTime/API.php
@@ -85,6 +85,7 @@ class Piwik_VisitTime_API
         $dataTable = $archive->getDataTableFromNumeric($metrics)->mergeChildren();
 
         // if there's no data for this report, don't bother w/ anything else
+        // TODO: with changes to getDataTableFromNumeric, this code would have to check if every row has 0 column values. is it really necessary? (assuming no for now)
         if ($dataTable->getRowsCount() == 0) {
             return $dataTable;
         }
diff --git a/plugins/VisitsSummary/API.php b/plugins/VisitsSummary/API.php
index 0864c517b5..9221d941bd 100644
--- a/plugins/VisitsSummary/API.php
+++ b/plugins/VisitsSummary/API.php
@@ -100,7 +100,7 @@ class Piwik_VisitsSummary_API
     {
         Piwik::checkUserHasViewAccess($idSite);
         $archive = Piwik_Archive::build($idSite, $period, $date, $segment);
-        $dataTable = $archive->getNumeric($toFetch);
+        $dataTable = $archive->getDataTableFromNumeric($toFetch);
         return $dataTable;
     }
 
@@ -142,8 +142,9 @@ class Piwik_VisitsSummary_API
     public function getSumVisitsLengthPretty($idSite, $period, $date, $segment = false)
     {
         $table = $this->getSumVisitsLength($idSite, $period, $date, $segment);
-        if ($table instanceof Piwik_DataTable_Array) {
-            $table->filter('ColumnCallbackReplace', array(0, array('Piwik', 'getPrettyTimeFromSeconds')));
+        if (is_object($table)) {
+            $table->filter('ColumnCallbackReplace',
+                array('sum_visit_length', array('Piwik', 'getPrettyTimeFromSeconds')));
         } else {
             $table = Piwik::getPrettyTimeFromSeconds($table);
         }