Faster flattening for many reports

8 files changed, 192 insertions, 79 deletions

diff --git a/plugins/Actions/API.php b/plugins/Actions/API.php
index 0ec50bceea..7204ddfa50 100644
--- a/plugins/Actions/API.php
+++ b/plugins/Actions/API.php
@@ -9,7 +9,6 @@
 namespace Piwik\Plugins\Actions;
 
 use Exception;
-use Piwik\API\Request;
 use Piwik\Archive;
 use Piwik\Common;
 use Piwik\DataTable;
@@ -84,18 +83,16 @@ class API extends \Piwik\Plugin\API
      * @param bool $expanded
      * @param bool|int $idSubtable
      * @param bool|int $depth
+     * @param bool|int $flat
      *
      * @return DataTable|DataTable\Map
      */
     public function getPageUrls($idSite, $period, $date, $segment = false, $expanded = false, $idSubtable = false,
-                                $depth = false, $flat= false)
+                                $depth = false, $flat = false)
     {
-        if ($flat && !$idSubtable) {
-            $expanded = true;
-        }
+        $dataTable = Archive::createDataTableFromArchive('Actions_actions_url', $idSite, $period, $date, $segment, $expanded, $flat, $idSubtable, $depth);
 
-        $dataTable = $this->getDataTableFromArchive('Actions_actions_url', $idSite, $period, $date, $segment, $expanded, $idSubtable, $depth);
-        $this->filterActionsDataTable($dataTable, $expanded);
+        $this->filterActionsDataTable($dataTable);
 
         return $dataTable;
     }
@@ -172,17 +169,19 @@ class API extends \Piwik\Plugin\API
 
     public function getPageUrl($pageUrl, $idSite, $period, $date, $segment = false)
     {
-        $callBackParameters = array('Actions_actions_url', $idSite, $period, $date, $segment, $expanded = false, $idSubtable = false);
+        $callBackParameters = array('Actions_actions_url', $idSite, $period, $date, $segment, $expanded = false, $flat = false, $idSubtable = null);
         $dataTable = $this->getFilterPageDatatableSearch($callBackParameters, $pageUrl, Action::TYPE_PAGE_URL);
         $this->addPageProcessedMetrics($dataTable);
         $this->filterActionsDataTable($dataTable);
         return $dataTable;
     }
 
-    public function getPageTitles($idSite, $period, $date, $segment = false, $expanded = false, $idSubtable = false)
+    public function getPageTitles($idSite, $period, $date, $segment = false, $expanded = false, $idSubtable = false, $flat = false)
     {
-        $dataTable = $this->getDataTableFromArchive('Actions_actions', $idSite, $period, $date, $segment, $expanded, $idSubtable);
-        $this->filterActionsDataTable($dataTable, $expanded);
+        $dataTable = Archive::createDataTableFromArchive('Actions_actions', $idSite, $period, $date, $segment, $expanded, $flat, $idSubtable);
+
+        $this->filterActionsDataTable($dataTable);
+
         return $dataTable;
     }
 
@@ -212,38 +211,38 @@ class API extends \Piwik\Plugin\API
 
     public function getPageTitle($pageName, $idSite, $period, $date, $segment = false)
     {
-        $callBackParameters = array('Actions_actions', $idSite, $period, $date, $segment, $expanded = false, $idSubtable = false);
+        $callBackParameters = array('Actions_actions', $idSite, $period, $date, $segment, $expanded = false, $flat = false, $idSubtable = null);
         $dataTable = $this->getFilterPageDatatableSearch($callBackParameters, $pageName, Action::TYPE_PAGE_TITLE);
         $this->addPageProcessedMetrics($dataTable);
         $this->filterActionsDataTable($dataTable);
         return $dataTable;
     }
 
-    public function getDownloads($idSite, $period, $date, $segment = false, $expanded = false, $idSubtable = false)
+    public function getDownloads($idSite, $period, $date, $segment = false, $expanded = false, $idSubtable = false, $flat = false)
     {
-        $dataTable = $this->getDataTableFromArchive('Actions_downloads', $idSite, $period, $date, $segment, $expanded, $idSubtable);
+        $dataTable = Archive::createDataTableFromArchive('Actions_downloads', $idSite, $period, $date, $segment, $expanded, $flat, $idSubtable);
         $this->filterActionsDataTable($dataTable, $expanded);
         return $dataTable;
     }
 
     public function getDownload($downloadUrl, $idSite, $period, $date, $segment = false)
     {
-        $callBackParameters = array('Actions_downloads', $idSite, $period, $date, $segment, $expanded = false, $idSubtable = false);
+        $callBackParameters = array('Actions_downloads', $idSite, $period, $date, $segment, $expanded = false, $flat = false, $idSubtable = null);
         $dataTable = $this->getFilterPageDatatableSearch($callBackParameters, $downloadUrl, Action::TYPE_DOWNLOAD);
         $this->filterActionsDataTable($dataTable);
         return $dataTable;
     }
 
-    public function getOutlinks($idSite, $period, $date, $segment = false, $expanded = false, $idSubtable = false)
+    public function getOutlinks($idSite, $period, $date, $segment = false, $expanded = false, $idSubtable = false, $flat = false)
     {
-        $dataTable = $this->getDataTableFromArchive('Actions_outlink', $idSite, $period, $date, $segment, $expanded, $idSubtable);
+        $dataTable = Archive::createDataTableFromArchive('Actions_outlink', $idSite, $period, $date, $segment, $expanded, $flat, $idSubtable);
         $this->filterActionsDataTable($dataTable, $expanded);
         return $dataTable;
     }
 
     public function getOutlink($outlinkUrl, $idSite, $period, $date, $segment = false)
     {
-        $callBackParameters = array('Actions_outlink', $idSite, $period, $date, $segment, $expanded = false, $idSubtable = false);
+        $callBackParameters = array('Actions_outlink', $idSite, $period, $date, $segment, $expanded = false, $flat = false, $idSubtable = null);
         $dataTable = $this->getFilterPageDatatableSearch($callBackParameters, $outlinkUrl, Action::TYPE_OUTLINK);
         $this->filterActionsDataTable($dataTable);
         return $dataTable;
@@ -254,6 +253,7 @@ class API extends \Piwik\Plugin\API
         $dataTable = $this->getSiteSearchKeywordsRaw($idSite, $period, $date, $segment);
         $dataTable->deleteColumn(PiwikMetrics::INDEX_SITE_SEARCH_HAS_NO_RESULT);
         $this->filterActionsDataTable($dataTable);
+        $dataTable->filter('ReplaceColumnNames');
         $this->addPagesPerSearchColumn($dataTable);
         return $dataTable;
     }
@@ -271,7 +271,7 @@ class API extends \Piwik\Plugin\API
 
     protected function getSiteSearchKeywordsRaw($idSite, $period, $date, $segment)
     {
-        $dataTable = $this->getDataTableFromArchive('Actions_sitesearch', $idSite, $period, $date, $segment, $expanded = false);
+        $dataTable = Archive::createDataTableFromArchive('Actions_sitesearch', $idSite, $period, $date, $segment, $expanded = false);
         return $dataTable;
     }
 
@@ -289,6 +289,7 @@ class API extends \Piwik\Plugin\API
         $dataTable->deleteRow(DataTable::ID_SUMMARY_ROW);
         $dataTable->deleteColumn(PiwikMetrics::INDEX_SITE_SEARCH_HAS_NO_RESULT);
         $this->filterActionsDataTable($dataTable);
+        $dataTable->filter('ReplaceColumnNames');
         $this->addPagesPerSearchColumn($dataTable);
         return $dataTable;
     }
@@ -337,6 +338,7 @@ class API extends \Piwik\Plugin\API
             }
         }
         $this->filterActionsDataTable($dataTable);
+        $dataTable->filter('ReplaceColumnNames');
         $this->addPagesPerSearchColumn($dataTable, $columnToRead = 'nb_actions');
         return $dataTable;
     }
@@ -366,7 +368,7 @@ class API extends \Piwik\Plugin\API
 
         if ($table === false) {
             // fetch the data table
-            $table = call_user_func_array(array($this, 'getDataTableFromArchive'), $callBackParameters);
+            $table = call_user_func_array('\Piwik\Archive::createDataTableFromArchive', $callBackParameters);
 
             if ($table instanceof DataTable\Map) {
                 // search an array of tables, e.g. when using date=last30
@@ -429,7 +431,7 @@ class API extends \Piwik\Plugin\API
 
             // match found on this level and more levels remaining: go deeper
             $idSubTable = $row->getIdSubDataTable();
-            $callBackParameters[6] = $idSubTable;
+            $callBackParameters[7] = $idSubTable;
 
             /**
              * @var \Piwik\Period $period
@@ -439,7 +441,7 @@ class API extends \Piwik\Plugin\API
                 $callBackParameters[3] = $period->getDateStart() . ',' . $period->getDateEnd();
             }
 
-            $table = call_user_func_array(array($this, 'getDataTableFromArchive'), $callBackParameters);
+            $table = call_user_func_array('\Piwik\Archive::createDataTableFromArchive', $callBackParameters);
             return $this->doFilterPageDatatableSearch($callBackParameters, $table, $searchTree);
         }
 
@@ -450,28 +452,15 @@ class API extends \Piwik\Plugin\API
      * Common filters for all Actions API
      *
      * @param DataTable|DataTable\Simple|DataTable\Map $dataTable
-     * @param bool $expanded
      */
-    protected function filterActionsDataTable($dataTable, $expanded = false)
+    private function filterActionsDataTable($dataTable)
     {
         // Must be applied before Sort in this case, since the DataTable can contain both int and strings indexes
         // (in the transition period between pre 1.2 and post 1.2 datatable structure)
-        $dataTable->filter('ReplaceColumnNames');
-        $dataTable->filter('Sort', array('nb_visits', 'desc', $naturalSort = false, $expanded));
-        $dataTable->filter(function (DataTable $dataTable) {
-            foreach ($dataTable->getRows() as $row) {
-                $url = $row->getMetadata('url');
-                if ($url) {
-                    $row->setMetadata('segmentValue', urldecode($url));
-                }
-            }
-        });
 
-        $dataTable->filter('GroupBy', array('label', function ($label) {
-            return urldecode($label);
-        }));
+        $dataTable->filter('Piwik\Plugins\Actions\DataTable\Filter\Actions');
 
-        $dataTable->queueFilter('ReplaceSummaryRowLabel');
+        return $dataTable;
     }
 
     /**
@@ -505,11 +494,6 @@ class API extends \Piwik\Plugin\API
         );
     }
 
-    protected function getDataTableFromArchive($name, $idSite, $period, $date, $segment, $expanded = false, $idSubtable = null, $depth = null)
-    {
-        return Archive::getDataTableFromArchive($name, $idSite, $period, $date, $segment, $expanded, $idSubtable, $depth);
-    }
-
     private function addPageProcessedMetrics(DataTable\DataTableInterface $dataTable)
     {
         $dataTable->filter(function (DataTable $table) {
diff --git a/plugins/Actions/DataTable/Filter/Actions.php b/plugins/Actions/DataTable/Filter/Actions.php
new file mode 100644
index 0000000000..71abe4190b
--- /dev/null
+++ b/plugins/Actions/DataTable/Filter/Actions.php
@@ -0,0 +1,53 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Actions\DataTable\Filter;
+
+use Piwik\DataTable\BaseFilter;
+use Piwik\DataTable\Row;
+use Piwik\DataTable;
+
+class Actions extends BaseFilter
+{
+    /**
+     * Constructor.
+     *
+     * @param DataTable $table The table to eventually filter.
+     */
+    public function __construct($table)
+    {
+        parent::__construct($table);
+    }
+
+    /**
+     * @param DataTable $table
+     */
+    public function filter($table)
+    {
+        $table->filter(function (DataTable $dataTable) {
+            foreach ($dataTable->getRows() as $row) {
+                $url = $row->getMetadata('url');
+                if ($url) {
+                    $row->setMetadata('segmentValue', urldecode($url));
+                }
+            }
+        });
+
+        // TODO can we remove this one again?
+        $table->queueFilter('GroupBy', array('label', function ($label) {
+            return urldecode($label);
+        }));
+
+        foreach ($table->getRows() as $row) {
+            $subtable = $row->getSubtable();
+            if ($subtable) {
+                $this->filter($subtable);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/plugins/Actions/Reports/Base.php b/plugins/Actions/Reports/Base.php
index 68bca741a3..6d7146ae73 100644
--- a/plugins/Actions/Reports/Base.php
+++ b/plugins/Actions/Reports/Base.php
@@ -22,6 +22,7 @@ abstract class Base extends \Piwik\Plugin\Report
     {
         $this->category = 'General_Actions';
         $this->processedMetrics = false;
+        $this->recursiveLabelSeparator = '/';
     }
 
     protected function addBaseDisplayProperties(ViewDataTable $view)
diff --git a/plugins/Events/API.php b/plugins/Events/API.php
index bb6584b777..7950cd2637 100644
--- a/plugins/Events/API.php
+++ b/plugins/Events/API.php
@@ -148,14 +148,17 @@ class API extends \Piwik\Plugin\API
         }
     }
 
-    protected function getDataTable($name, $idSite, $period, $date, $segment, $expanded = false, $idSubtable = null, $secondaryDimension = false)
+    protected function getDataTable($name, $idSite, $period, $date, $segment, $expanded = false, $idSubtable = null, $secondaryDimension = false, $flat = false)
     {
         Piwik::checkUserHasViewAccess($idSite);
         $this->checkSecondaryDimension($name, $secondaryDimension);
         $recordName = $this->getRecordNameForAction($name, $secondaryDimension);
-        $dataTable = Archive::getDataTableFromArchive($recordName, $idSite, $period, $date, $segment, $expanded, $idSubtable);
 
-        if (empty($idSubtable)) {
+        $dataTable = Archive::createDataTableFromArchive($recordName, $idSite, $period, $date, $segment, $expanded, $flat, $idSubtable);
+
+        if ($flat) {
+            $dataTable->filterSubtables('Piwik\Plugins\Events\DataTable\Filter\ReplaceEventNameNotSet');
+        } else {
             $dataTable->filter('AddSegmentValue', array(function ($label) {
                 if ($label === Archiver::EVENT_NAME_NOT_SET) {
                     return false;
@@ -165,23 +168,24 @@ class API extends \Piwik\Plugin\API
             }));
         }
 
-        $this->filterDataTable($dataTable);
+        $dataTable->filter('Piwik\Plugins\Events\DataTable\Filter\ReplaceEventNameNotSet');
+
         return $dataTable;
     }
 
-    public function getCategory($idSite, $period, $date, $segment = false, $expanded = false, $secondaryDimension = false)
+    public function getCategory($idSite, $period, $date, $segment = false, $expanded = false, $secondaryDimension = false, $flat = false)
     {
-        return $this->getDataTable(__FUNCTION__, $idSite, $period, $date, $segment, $expanded, $idSubtable = false, $secondaryDimension);
+        return $this->getDataTable(__FUNCTION__, $idSite, $period, $date, $segment, $expanded, $idSubtable = false, $secondaryDimension, $flat);
     }
 
-    public function getAction($idSite, $period, $date, $segment = false, $expanded = false, $secondaryDimension = false)
+    public function getAction($idSite, $period, $date, $segment = false, $expanded = false, $secondaryDimension = false, $flat = false)
     {
-        return $this->getDataTable(__FUNCTION__, $idSite, $period, $date, $segment, $expanded, $idSubtable = false, $secondaryDimension);
+        return $this->getDataTable(__FUNCTION__, $idSite, $period, $date, $segment, $expanded, $idSubtable = false, $secondaryDimension, $flat);
     }
 
-    public function getName($idSite, $period, $date, $segment = false, $expanded = false, $secondaryDimension = false)
+    public function getName($idSite, $period, $date, $segment = false, $expanded = false, $secondaryDimension = false, $flat = false)
     {
-        return $this->getDataTable(__FUNCTION__, $idSite, $period, $date, $segment, $expanded, $idSubtable = false, $secondaryDimension);
+        return $this->getDataTable(__FUNCTION__, $idSite, $period, $date, $segment, $expanded, $idSubtable = false, $secondaryDimension, $flat);
     }
 
     public function getActionFromCategoryId($idSite, $period, $date, $idSubtable, $segment = false)
@@ -213,20 +217,4 @@ class API extends \Piwik\Plugin\API
     {
         return $this->getDataTable(__FUNCTION__, $idSite, $period, $date, $segment, $expanded = false, $idSubtable);
     }
-
-    /**
-     * @param DataTable $dataTable
-     */
-    protected function filterDataTable($dataTable)
-    {
-        $dataTable->filter('Sort', array(Metrics::INDEX_NB_VISITS));
-        $dataTable->queueFilter('ReplaceColumnNames');
-        $dataTable->queueFilter('ReplaceSummaryRowLabel');
-        $dataTable->filter(function (DataTable $table) {
-            $row = $table->getRowFromLabel(Archiver::EVENT_NAME_NOT_SET);
-            if ($row) {
-                $row->setColumn('label', Piwik::translate('General_NotDefined', Piwik::translate('Events_EventName')));
-            }
-        });
-    }
 }
\ No newline at end of file
diff --git a/plugins/Events/DataTable/Filter/ReplaceEventNameNotSet.php b/plugins/Events/DataTable/Filter/ReplaceEventNameNotSet.php
new file mode 100644
index 0000000000..a07695b5f8
--- /dev/null
+++ b/plugins/Events/DataTable/Filter/ReplaceEventNameNotSet.php
@@ -0,0 +1,39 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Events\DataTable\Filter;
+
+use Piwik\DataTable\BaseFilter;
+use Piwik\DataTable\Row;
+use Piwik\DataTable;
+use Piwik\Piwik;
+use Piwik\Plugins\Events\Archiver;
+
+class ReplaceEventNameNotSet extends BaseFilter
+{
+    /**
+     * Constructor.
+     *
+     * @param DataTable $table The table to eventually filter.
+     */
+    public function __construct($table)
+    {
+        parent::__construct($table);
+    }
+
+    /**
+     * @param DataTable $table
+     */
+    public function filter($table)
+    {
+        $row = $table->getRowFromLabel(Archiver::EVENT_NAME_NOT_SET);
+        if ($row) {
+            $row->setColumn('label', Piwik::translate('General_NotDefined', Piwik::translate('Events_EventName')));
+        }
+    }
+}
\ No newline at end of file
diff --git a/plugins/Referrers/API.php b/plugins/Referrers/API.php
index 4ac28978e3..aa1a3e0419 100644
--- a/plugins/Referrers/API.php
+++ b/plugins/Referrers/API.php
@@ -43,7 +43,6 @@ class API extends \Piwik\Plugin\API
     protected function getDataTable($name, $idSite, $period, $date, $segment, $expanded = false, $idSubtable = null)
     {
         $dataTable = Archive::getDataTableFromArchive($name, $idSite, $period, $date, $segment, $expanded, $idSubtable);
-        $dataTable->filter('Sort', array(Metrics::INDEX_NB_VISITS, 'desc', $naturalSort = false, $expanded));
         $dataTable->queueFilter('ReplaceColumnNames');
         return $dataTable;
     }
@@ -296,22 +295,24 @@ class API extends \Piwik\Plugin\API
         return $dataTable;
     }
 
-    public function getWebsites($idSite, $period, $date, $segment = false, $expanded = false)
+    public function getWebsites($idSite, $period, $date, $segment = false, $expanded = false, $flat = false)
     {
-        $dataTable = $this->getDataTable(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded);
-        $dataTable->filter('AddSegmentByLabel', array('referrerName'));
+        $dataTable = Archive::createDataTableFromArchive(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded, $flat, $idSubtable = null);
+
+        if ($flat) {
+            $dataTable->filterSubtables('Piwik\Plugins\Referrers\DataTable\Filter\UrlsFromWebsiteId');
+        } else {
+            $dataTable->filter('AddSegmentByLabel', array('referrerName'));
+        }
+
         return $dataTable;
     }
 
     public function getUrlsFromWebsiteId($idSite, $period, $date, $idSubtable, $segment = false)
     {
         $dataTable = $this->getDataTable(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded = false, $idSubtable);
-        // the htmlspecialchars_decode call is for BC for before 1.1
-        // as the Referrer URL was previously encoded in the log tables, but is now recorded raw
-        $dataTable->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', function ($label) {
-            return htmlspecialchars_decode($label);
-        }));
-        $dataTable->queueFilter('ColumnCallbackReplace', array('label', __NAMESPACE__ . '\getPathFromUrl'));
+        $dataTable->filter('Piwik\Plugins\Referrers\DataTable\Filter\UrlsFromWebsiteId');
+
         return $dataTable;
     }
 
diff --git a/plugins/Referrers/DataTable/Filter/UrlsFromWebsiteId.php b/plugins/Referrers/DataTable/Filter/UrlsFromWebsiteId.php
new file mode 100644
index 0000000000..c35d60b463
--- /dev/null
+++ b/plugins/Referrers/DataTable/Filter/UrlsFromWebsiteId.php
@@ -0,0 +1,46 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Referrers\DataTable\Filter;
+
+use Piwik\DataTable\BaseFilter;
+use Piwik\DataTable\Row;
+use Piwik\DataTable;
+
+class UrlsFromWebsiteId extends BaseFilter
+{
+    /**
+     * Constructor.
+     *
+     * @param DataTable $table The table to eventually filter.
+     */
+    public function __construct($table)
+    {
+        parent::__construct($table);
+    }
+
+    /**
+     * @param DataTable $table
+     */
+    public function filter($table)
+    {
+        // the htmlspecialchars_decode call is for BC for before 1.1
+        // as the Referrer URL was previously encoded in the log tables, but is now recorded raw
+        $table->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', function ($label) {
+            return htmlspecialchars_decode($label);
+        }));
+        $table->queueFilter('ColumnCallbackReplace', array('label', 'Piwik\Plugins\Referrers\getPathFromUrl'));
+
+        foreach ($table->getRows() as $row) {
+            $subtable = $row->getSubtable();
+            if ($subtable) {
+                $this->filter($subtable);
+            }
+        }
+    }
+}
\ No newline at end of file
diff --git a/plugins/Referrers/Reports/GetWebsites.php b/plugins/Referrers/Reports/GetWebsites.php
index e394eb43d9..18f9336a63 100644
--- a/plugins/Referrers/Reports/GetWebsites.php
+++ b/plugins/Referrers/Reports/GetWebsites.php
@@ -21,6 +21,7 @@ class GetWebsites extends Base
         $this->dimension     = new Website();
         $this->name          = Piwik::translate('CorePluginsAdmin_Websites');
         $this->documentation = Piwik::translate('Referrers_WebsitesReportDocumentation', '<br />');
+        $this->recursiveLabelSeparator = '/';
         $this->actionToLoadSubTables = 'getUrlsFromWebsiteId';
         $this->hasGoalMetrics = true;
         $this->order = 5;