Merge pull request #7693 from piwik/6809_2

All websites dashboard - improve performance when thousands of websites

14 files changed, 1113 insertions, 423 deletions

diff --git a/plugins/MultiSites/.gitignore b/plugins/MultiSites/.gitignore
new file mode 100644
index 0000000000..c8c9480010
--- /dev/null
+++ b/plugins/MultiSites/.gitignore
@@ -0,0 +1 @@
+tests/System/processed/*xml
\ No newline at end of file
diff --git a/plugins/MultiSites/API.php b/plugins/MultiSites/API.php
index c85dc94024..df92877416 100755
--- a/plugins/MultiSites/API.php
+++ b/plugins/MultiSites/API.php
@@ -14,6 +14,7 @@ use Piwik\Archive;
 use Piwik\Common;
 use Piwik\Container\StaticContainer;
 use Piwik\DataTable;
+use Piwik\DataTable\Row;
 use Piwik\Period\Range;
 use Piwik\Piwik;
 use Piwik\Plugins\Goals\Archiver;
@@ -81,9 +82,10 @@ class API extends \Piwik\Plugin\API
      *                                        Only used when a scheduled task is running
      * @param bool|string $enhanced When true, return additional goal & ecommerce metrics
      * @param bool|string $pattern If specified, only the website which names (or site ID) match the pattern will be returned using SitesManager.getPatternMatchSites
+     * @param array $showColumns If specified, only the requested columns will be fetched
      * @return DataTable
      */
-    public function getAll($period, $date, $segment = false, $_restrictSitesToLogin = false, $enhanced = false, $pattern = false)
+    public function getAll($period, $date, $segment = false, $_restrictSitesToLogin = false, $enhanced = false, $pattern = false, $showColumns = array())
     {
         Piwik::checkUserHasSomeViewAccess();
 
@@ -100,7 +102,8 @@ class API extends \Piwik\Plugin\API
             $segment,
             $_restrictSitesToLogin,
             $enhanced,
-            $multipleWebsitesRequested = true
+            $multipleWebsitesRequested = true,
+            $showColumns
         );
     }
 
@@ -185,7 +188,8 @@ class API extends \Piwik\Plugin\API
             $segment,
             $_restrictSitesToLogin,
             $enhanced,
-            $multipleWebsitesRequested = false
+            $multipleWebsitesRequested = false,
+            $showColumns = array()
         );
     }
 
@@ -197,7 +201,7 @@ class API extends \Piwik\Plugin\API
         return $sites;
     }
 
-    private function buildDataTable($sitesToProblablyAdd, $period, $date, $segment, $_restrictSitesToLogin, $enhanced, $multipleWebsitesRequested)
+    private function buildDataTable($sitesToProblablyAdd, $period, $date, $segment, $_restrictSitesToLogin, $enhanced, $multipleWebsitesRequested, $showColumns)
     {
         $idSites = array();
         if (!empty($sitesToProblablyAdd)) {
@@ -221,6 +225,10 @@ class API extends \Piwik\Plugin\API
         $apiECommerceMetrics = array();
         $apiMetrics = API::getApiMetrics($enhanced);
         foreach ($apiMetrics as $metricName => $metricSettings) {
+            if (!empty($showColumns) && !in_array($metricName, $showColumns)) {
+                unset($apiMetrics[$metricName]);
+                continue;
+            }
             $fieldsToGet[] = $metricSettings[self::METRIC_RECORD_NAME_KEY];
             $columnNameRewrites[$metricSettings[self::METRIC_RECORD_NAME_KEY]] = $metricName;
 
@@ -229,25 +237,11 @@ class API extends \Piwik\Plugin\API
             }
         }
 
-        // get the data
-        // $dataTable instanceOf Set
-        $dataTable = $archive->getDataTableFromNumeric($fieldsToGet);
-
-        if ($multipleWebsitesRequested && count($idSites) === 1 && Range::isMultiplePeriod($date, $period)) {
-        } else {
-            $dataTable = $this->mergeDataTableMapAndPopulateLabel($idSites, $multipleWebsitesRequested, $dataTable);
-        }
-
-        if ($dataTable instanceof DataTable\Map) {
-            foreach ($dataTable->getDataTables() as $table) {
-                $this->addMissingWebsites($table, $fieldsToGet, $sitesToProblablyAdd);
-            }
-        } else {
-            $this->addMissingWebsites($dataTable, $fieldsToGet, $sitesToProblablyAdd);
-        }
+        $dataTable = $archive->getDataTableFromNumericAndMergeChildren($fieldsToGet);
 
-        // calculate total visits/actions/revenue
-        $this->setMetricsTotalsMetadata($dataTable, $apiMetrics);
+        $this->populateLabel($dataTable);
+        $totalMetrics = $this->preformatApiMetricsForTotalsCalculation($apiMetrics);
+        $this->setMetricsTotalsMetadata($dataTable, $totalMetrics);
 
         // if the period isn't a range & a lastN/previousN date isn't used, we get the same
         // data for the last period to show the evolution of visits/actions/revenue
@@ -263,23 +257,16 @@ class API extends \Piwik\Plugin\API
             }
 
             $pastArchive = Archive::build($idSites, $period, $strLastDate, $segment, $_restrictSitesToLogin);
+            $pastData = $pastArchive->getDataTableFromNumericAndMergeChildren($fieldsToGet);
 
-            $pastData = $pastArchive->getDataTableFromNumeric($fieldsToGet);
-
-            if ($multipleWebsitesRequested && count($idSites) === 1 && Range::isMultiplePeriod($date, $period)) {
-
-            } else {
-                $pastData = $this->mergeDataTableMapAndPopulateLabel($idSites, $multipleWebsitesRequested, $pastData);
-            }
-
-            // use past data to calculate evolution percentages
+            $this->populateLabel($pastData); // labels are needed to calculate evolution
             $this->calculateEvolutionPercentages($dataTable, $pastData, $apiMetrics);
+            $this->setPastTotalVisitsMetadata($dataTable, $pastData);
         }
 
-        // move the site id to a metadata column
-        $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'group', array('\Piwik\Site', 'getGroupFor'), array()));
-        $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'main_url', array('\Piwik\Site', 'getMainUrlFor'), array()));
-        $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'idsite'));
+        // move the site id to a metadata column 
+        $dataTable->queueFilter('MetadataCallbackAddMetadata', array('idsite', 'group', array('\Piwik\Site', 'getGroupFor'), array()));
+        $dataTable->queueFilter('MetadataCallbackAddMetadata', array('idsite', 'main_url', array('\Piwik\Site', 'getMainUrlFor'), array()));
 
         // set the label of each row to the site name
         if ($multipleWebsitesRequested) {
@@ -420,6 +407,17 @@ class API extends \Piwik\Plugin\API
         return $metrics;
     }
 
+    private function preformatApiMetricsForTotalsCalculation($apiMetrics)
+    {
+        $metrics = array();
+        foreach ($apiMetrics as $label => $metricsInfo) {
+            $totalMetadataName = self::getTotalMetadataName($label);
+            $metrics[$totalMetadataName] = $metricsInfo[self::METRIC_RECORD_NAME_KEY];
+        }
+
+        return $metrics;
+    }
+
     /**
      * Sets the total visits, actions & revenue for a DataTable returned by
      * $this->buildDataTable.
@@ -436,92 +434,63 @@ class API extends \Piwik\Plugin\API
             }
         } else {
             $totals = array();
-            foreach ($apiMetrics as $label => $metricInfo) {
-                $totalMetadataName = self::getTotalMetadataName($label);
-                $totals[$totalMetadataName] = 0;
+            foreach ($apiMetrics as $label => $recordName) {
+                $totals[$label] = 0;
             }
 
             foreach ($dataTable->getRows() as $row) {
-                foreach ($apiMetrics as $label => $metricInfo) {
-                    $totalMetadataName = self::getTotalMetadataName($label);
-                    $totals[$totalMetadataName] += $row->getColumn($metricInfo[self::METRIC_RECORD_NAME_KEY]);
+                foreach ($apiMetrics as $totalMetadataName => $recordName) {
+                    $totals[$totalMetadataName] += $row->getColumn($recordName);
                 }
             }
 
-            foreach ($totals as $name => $value) {
-                $dataTable->setMetadata($name, $value);
-            }
+            $dataTable->setMetadataValues($totals);
         }
     }
 
-    private static function getTotalMetadataName($name)
-    {
-        return 'total_' . $name;
-    }
-
-    private static function getLastPeriodMetadataName($name)
-    {
-        return 'last_period_' . $name;
-    }
-
     /**
-     * @param DataTable|DataTable\Map $dataTable
-     * @param $fieldsToGet
-     * @param $sitesToProblablyAdd
+     * Sets the number of total visits in tha pastTable on the dataTable as metadata.
+     *
+     * @param DataTable $dataTable
+     * @param DataTable $pastTable
      */
-    private function addMissingWebsites($dataTable, $fieldsToGet, $sitesToProblablyAdd)
+    private function setPastTotalVisitsMetadata($dataTable, $pastTable)
     {
-        $siteIdsInDataTable = array();
-        foreach ($dataTable->getRows() as $row) {
-            /** @var DataTable\Row $row */
-            $siteIdsInDataTable[] = $row->getColumn('label');
-        }
+        if ($pastTable instanceof DataTable) {
+            $total  = 0;
+            $metric = 'nb_visits';
 
-        foreach ($sitesToProblablyAdd as $site) {
-            if (!in_array($site['idsite'], $siteIdsInDataTable)) {
-                $siteRow = array_combine($fieldsToGet, array_pad(array(), count($fieldsToGet), 0));
-                $siteRow['label'] = (int) $site['idsite'];
-                $dataTable->addRowFromSimpleArray($siteRow);
+            foreach ($pastTable->getRows() as $row) {
+                $total += $row->getColumn($metric);
             }
+
+            $dataTable->setMetadata(self::getTotalMetadataName($metric . '_lastdate'), $total);
         }
     }
 
-    private function removeEcommerceRelatedMetricsOnNonEcommercePiwikSites($dataTable, $apiECommerceMetrics)
+    private static function getTotalMetadataName($name)
     {
-        // $dataTableRows instanceOf Row[]
-        $dataTableRows = $dataTable->getRows();
-
-        foreach ($dataTableRows as $dataTableRow) {
-            $siteId = $dataTableRow->getColumn('label');
-            if (!Site::isEcommerceEnabledFor($siteId)) {
-                foreach ($apiECommerceMetrics as $metricSettings) {
-                    $dataTableRow->deleteColumn($metricSettings[self::METRIC_RECORD_NAME_KEY]);
-                    $dataTableRow->deleteColumn($metricSettings[self::METRIC_EVOLUTION_COL_NAME_KEY]);
-                }
-            }
-        }
+        return 'total_' . $name;
     }
 
-    private function mergeDataTableMapAndPopulateLabel($idSitesOrIdSite, $multipleWebsitesRequested, $dataTable)
+    private static function getLastPeriodMetadataName($name)
     {
-        // get rid of the DataTable\Map that is created by the IndexedBySite archive type
-        if ($dataTable instanceof DataTable\Map && $multipleWebsitesRequested) {
-
-            return $dataTable->mergeChildren();
-
-        } else {
-
-            if (!$dataTable instanceof DataTable\Map && $dataTable->getRowsCount() > 0) {
-
-                $firstSite = is_array($idSitesOrIdSite) ? reset($idSitesOrIdSite) : $idSitesOrIdSite;
-
-                $firstDataTableRow = $dataTable->getFirstRow();
+        return 'last_period_' . $name;
+    }
 
-                $firstDataTableRow->setColumn('label', $firstSite);
+    private function populateLabel($dataTable)
+    {
+        $dataTable->filter(function (DataTable $table) {
+            foreach ($table->getRowsWithoutSummaryRow() as $row) {
+                $row->setColumn('label', $row->getMetadata('idsite'));
             }
-        }
-
-        return $dataTable;
+        });
+        // make sure label column is always first column
+        $dataTable->queueFilter(function (DataTable $table) {
+            foreach ($table->getRowsWithoutSummaryRow() as $row) {
+                $row->setColumns(array_merge(array('label' => $row->getColumn('label')), $row->getColumns()));
+            }
+        });
     }
 
     private function isEcommerceEvolutionMetric($metricSettings)
@@ -532,4 +501,4 @@ class API extends \Piwik\Plugin\API
             self::ECOMMERCE_REVENUE_METRIC . '_evolution'
         ));
     }
-}
\ No newline at end of file
+}
diff --git a/plugins/MultiSites/Columns/Metrics/EcommerceOnlyEvolutionMetric.php b/plugins/MultiSites/Columns/Metrics/EcommerceOnlyEvolutionMetric.php
index cee88af3f5..e43504154e 100644
--- a/plugins/MultiSites/Columns/Metrics/EcommerceOnlyEvolutionMetric.php
+++ b/plugins/MultiSites/Columns/Metrics/EcommerceOnlyEvolutionMetric.php
@@ -36,13 +36,13 @@ class EcommerceOnlyEvolutionMetric extends EvolutionMetric
 
         // if the site this is for doesn't support ecommerce & this is for the revenue_evolution column,
         // we don't add the new column
-        if (($currentValue === false
-            || !$this->isRevenueEvolution)
-            && !Site::isEcommerceEnabledFor($row->getColumn('label'))
-        ) {
-            $row->deleteColumn($columnName);
+        if ($currentValue === false || !$this->isRevenueEvolution) {
+            $idSite = $row->getMetadata('idsite');
+            if (!$idSite || !Site::isEcommerceEnabledFor($idSite)) {
+                $row->deleteColumn($columnName);
 
-            return false;
+                return false;
+            }
         }
 
         return parent::compute($row);
diff --git a/plugins/MultiSites/Controller.php b/plugins/MultiSites/Controller.php
index 647ae147dc..a59e8a4cdb 100644
--- a/plugins/MultiSites/Controller.php
+++ b/plugins/MultiSites/Controller.php
@@ -8,10 +8,15 @@
  */
 namespace Piwik\Plugins\MultiSites;
 
+use Piwik\API\Request;
+use Piwik\API\ResponseBuilder;
 use Piwik\Common;
 use Piwik\Config;
 use Piwik\Date;
 use Piwik\Period;
+use Piwik\DataTable;
+use Piwik\DataTable\Row;
+use Piwik\DataTable\Row\DataTableSummaryRow;
 use Piwik\Piwik;
 use Piwik\Translation\Translator;
 use Piwik\View;
@@ -40,6 +45,34 @@ class Controller extends \Piwik\Plugin\Controller
         return $this->getSitesInfo($isWidgetized = true);
     }
 
+    public function getAllWithGroups()
+    {
+        Piwik::checkUserHasSomeViewAccess();
+
+        $period  = Common::getRequestVar('period', null, 'string');
+        $date    = Common::getRequestVar('date', null, 'string');
+        $segment = Common::getRequestVar('segment', false, 'string');
+        $pattern = Common::getRequestVar('pattern', '', 'string');
+        $limit   = Common::getRequestVar('filter_limit', 0, 'int');
+        $segment = $segment ?: false;
+        $request = $_GET + $_POST;
+
+        $dashboard = new Dashboard($period, $date, $segment);
+
+        if ($pattern !== '') {
+            $dashboard->search(strtolower($pattern));
+        }
+
+        $response = array(
+            'numSites' => $dashboard->getNumSites(),
+            'totals'   => $dashboard->getTotals(),
+            'lastDate' => $dashboard->getLastDate(),
+            'sites'    => $dashboard->getSites($request, $limit)
+        );
+
+        return json_encode($response);
+    }
+
     public function getSitesInfo($isWidgetized = false)
     {
         Piwik::checkUserHasSomeViewAccess();
diff --git a/plugins/MultiSites/Dashboard.php b/plugins/MultiSites/Dashboard.php
new file mode 100644
index 0000000000..d33d07e5ec
--- /dev/null
+++ b/plugins/MultiSites/Dashboard.php
@@ -0,0 +1,298 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\MultiSites;
+
+use Piwik\API\ResponseBuilder;
+use Piwik\Config;
+use Piwik\Metrics\Formatter;
+use Piwik\Period;
+use Piwik\DataTable;
+use Piwik\DataTable\Row;
+use Piwik\DataTable\Row\DataTableSummaryRow;
+use Piwik\Plugins\API\ProcessedReport;
+use Piwik\Site;
+use Piwik\View;
+
+/**
+ * Fetches and formats the response of `MultiSites.getAll` in a way that it can be used by the All Websites AngularJS
+ * widget. Eg sites are moved into groups if one is assigned, stats are calculated for groups, etc.
+ */
+class Dashboard
+{
+    /** @var DataTable */
+    private $sitesByGroup;
+
+    /**
+     * @var int
+     */
+    private $numSites = 0;
+
+    /**
+     * @param string $period
+     * @param string $date
+     * @param string|false $segment
+     */
+    public function __construct($period, $date, $segment)
+    {
+        $sites = API::getInstance()->getAll($period, $date, $segment, $_restrictSitesToLogin = false,
+                                            $enhanced = true, $searchTerm = false,
+                                            $showColumns = array('nb_visits', 'nb_pageviews', 'revenue'));
+        $sites->deleteRow(DataTable::ID_SUMMARY_ROW);
+        $sites->filter(function (DataTable $table) {
+            foreach ($table->getRows() as $row) {
+                $idSite = $row->getColumn('label');
+                $site   = Site::getSite($idSite);
+                // we cannot queue label and group as we might need them for search!
+                $row->setColumn('label', $site['name']);
+                $row->setMetadata('group', $site['group']);
+            }
+        });
+
+        $this->setSitesTable($sites);
+    }
+
+    public function setSitesTable(DataTable $sites)
+    {
+        $this->numSites     = $sites->getRowsCount();
+        $this->sitesByGroup = $this->moveSitesHavingAGroupIntoSubtables($sites);
+    }
+
+    public function getSites($request, $limit)
+    {
+        $request['filter_limit'] = $limit;
+
+        $sitesExpanded = $this->convertDataTableToArrayAndApplyFilters($this->sitesByGroup, $request);
+        $sitesFlat     = $this->makeSitesFlat($sitesExpanded);
+        $sitesFlat     = $this->applyLimitIfNeeded($sitesFlat, $limit);
+        $sitesFlat     = $this->enrichValues($sitesFlat);
+
+        return $sitesFlat;
+    }
+
+    public function getTotals()
+    {
+        return array(
+            'nb_pageviews'       => $this->sitesByGroup->getMetadata('total_nb_pageviews'),
+            'nb_visits'          => $this->sitesByGroup->getMetadata('total_nb_visits'),
+            'revenue'            => $this->sitesByGroup->getMetadata('total_revenue'),
+            'nb_visits_lastdate' => $this->sitesByGroup->getMetadata('total_nb_visits_lastdate') ? : 0,
+        );
+    }
+
+    public function getNumSites()
+    {
+        return $this->numSites;
+    }
+
+    public function search($pattern)
+    {
+        $this->nestedSearch($this->sitesByGroup, $pattern);
+
+        $this->numSites = $this->sitesByGroup->getRowsCountRecursive();
+    }
+
+    private function nestedSearch(DataTable $sitesByGroup, $pattern)
+    {
+        foreach ($sitesByGroup->getRows() as $index => $site) {
+
+            $label = strtolower($site->getColumn('label'));
+            $labelMatches = false !== strpos($label, $pattern);
+
+            if ($site->getMetadata('isGroup')) {
+                $subtable = $site->getSubtable();
+                $this->nestedSearch($subtable, $pattern);
+
+                if (!$labelMatches && !$subtable->getRowsCount()) {
+                    // we keep the group if at least one site within the group matches the pattern
+                    $sitesByGroup->deleteRow($index);
+                }
+
+            } elseif (!$labelMatches) {
+                $group = $site->getMetadata('group');
+
+                if (!$group || false === strpos(strtolower($group), $pattern)) {
+                    $sitesByGroup->deleteRow($index);
+                }
+            }
+        }
+    }
+
+    /**
+     * @return string
+     */
+    public function getLastDate()
+    {
+        $lastPeriod = $this->sitesByGroup->getMetadata('last_period_date');
+
+        if (!empty($lastPeriod)) {
+            $lastPeriod = $lastPeriod->toString();
+        } else {
+            $lastPeriod = '';
+        }
+
+        return $lastPeriod;
+    }
+
+    private function convertDataTableToArrayAndApplyFilters(DataTable $table, $request)
+    {
+        $request['serialize'] = 0;
+        $request['expanded'] = 1;
+        $request['totals'] = 0;
+        $request['format_metrics'] = 1;
+
+        // filter_sort_column does not work correctly is a bug in MultiSites.getAll
+        if (!empty($request['filter_sort_column']) && $request['filter_sort_column'] === 'nb_pageviews') {
+            $request['filter_sort_column'] = 'Actions_nb_pageviews';
+        } elseif (!empty($request['filter_sort_column']) && $request['filter_sort_column'] === 'revenue') {
+            $request['filter_sort_column'] = 'Goal_revenue';
+        }
+
+        $responseBuilder = new ResponseBuilder('php', $request);
+        $rows = $responseBuilder->getResponse($table, 'MultiSites', 'getAll');
+
+        return $rows;
+    }
+
+    private function moveSitesHavingAGroupIntoSubtables(DataTable $sites)
+    {
+        /** @var DataTableSummaryRow[] $groups */
+        $groups = array();
+
+        $sitesByGroup = $this->makeCloneOfDataTableSites($sites);
+        $sitesByGroup->enableRecursiveFilters(); // we need to make sure filters get applied to subtables (groups)
+
+        foreach ($sites->getRows() as $site) {
+
+            $group = $site->getMetadata('group');
+
+            if (!empty($group) && !array_key_exists($group, $groups)) {
+                $row = new DataTableSummaryRow();
+                $row->setColumn('label', $group);
+                $row->setMetadata('isGroup', 1);
+                $row->setSubtable($this->createGroupSubtable($sites));
+                $sitesByGroup->addRow($row);
+
+                $groups[$group] = $row;
+            }
+
+            if (!empty($group)) {
+                $groups[$group]->getSubtable()->addRow($site);
+            } else {
+                $sitesByGroup->addRow($site);
+            }
+        }
+
+        foreach ($groups as $group) {
+            // we need to recalculate as long as all rows are there, as soon as some rows are removed
+            // we can no longer recalculate the correct value. We might even calculate values for groups
+            // that are not returned. If this becomes a problem we need to keep a copy of this to recalculate
+            // only actual returned groups.
+            $group->recalculate();
+        }
+        
+        return $sitesByGroup;
+    }
+
+    private function createGroupSubtable(DataTable $sites)
+    {
+        $table = new DataTable();
+        $processedMetrics = $sites->getMetadata(DataTable::EXTRA_PROCESSED_METRICS_METADATA_NAME);
+        $table->setMetadata(DataTable::EXTRA_PROCESSED_METRICS_METADATA_NAME, $processedMetrics);
+
+        return $table;
+    }
+
+    private function makeCloneOfDataTableSites(DataTable $sites)
+    {
+        $sitesByGroup = $sites->getEmptyClone(true);
+        // we handle them ourselves for faster performance etc. This way we also avoid to apply them twice.
+        $sitesByGroup->disableFilter('ColumnCallbackReplace');
+        $sitesByGroup->disableFilter('MetadataCallbackAddMetadata');
+
+        return $sitesByGroup;
+    }
+
+    /**
+     * Makes sure to not have any subtables anymore.
+     * So if $sites is
+     * array(
+     *    site1
+     *    site2
+     *        subtable => site3
+     *                    site4
+     *                    site5
+     *    site6
+     *    site7
+     * )
+     *
+     * it will return
+     *
+     * array(
+     *    site1
+     *    site2
+     *    site3
+     *    site4
+     *    site5
+     *    site6
+     *    site7
+     * )
+     *
+     * @param $sites
+     * @return array
+     */
+    private function makeSitesFlat($sites)
+    {
+        $flatSites = array();
+
+        foreach ($sites as $site) {
+            if (!empty($site['subtable'])) {
+                if (isset($site['idsubdatatable'])) {
+                    unset($site['idsubdatatable']);
+                }
+
+                $subtable = $site['subtable'];
+                unset($site['subtable']);
+                $flatSites[] = $site;
+                foreach ($subtable as $siteWithinGroup) {
+                    $flatSites[] = $siteWithinGroup;
+                }
+            } else {
+                $flatSites[] = $site;
+            }
+        }
+
+        return $flatSites;
+    }
+
+    private function applyLimitIfNeeded($sites, $limit)
+    {
+        // why do we need to apply a limit again? because we made sitesFlat and it may contain many more sites now
+        if ($limit > 0) {
+            $sites = array_slice($sites, 0, $limit);
+        }
+
+        return $sites;
+    }
+
+    private function enrichValues($sites)
+    {
+        $formatter = new Formatter();
+
+        foreach ($sites as &$site) {
+            if (!isset($site['idsite'])) {
+                continue;
+            }
+
+            $site['revenue']  = $formatter->getPrettyMoney($site['revenue'], $site['idsite']);
+            $site['main_url'] = Site::getMainUrlFor($site['idsite']);
+        }
+
+        return $sites;
+    }
+}
diff --git a/plugins/MultiSites/MultiSites.php b/plugins/MultiSites/MultiSites.php
index f3f62ecae4..c69f174d84 100644
--- a/plugins/MultiSites/MultiSites.php
+++ b/plugins/MultiSites/MultiSites.php
@@ -68,13 +68,13 @@ class MultiSites extends \Piwik\Plugin
         $translations[] = 'MultiSites_LoadingWebsites';
         $translations[] = 'General_ErrorRequest';
         $translations[] = 'General_Pagination';
+        $translations[] = 'General_ClickToSearch';
     }
 
     public function getJsFiles(&$jsFiles)
     {
         $jsFiles[] = "plugins/MultiSites/angularjs/dashboard/dashboard-model.service.js";
         $jsFiles[] = "plugins/MultiSites/angularjs/dashboard/dashboard.controller.js";
-        $jsFiles[] = "plugins/MultiSites/angularjs/dashboard/dashboard-group.filter.js";
         $jsFiles[] = "plugins/MultiSites/angularjs/dashboard/dashboard.directive.js";
         $jsFiles[] = "plugins/MultiSites/angularjs/site/site.controller.js";
         $jsFiles[] = "plugins/MultiSites/angularjs/site/site.directive.js";
diff --git a/plugins/MultiSites/angularjs/dashboard/dashboard-group.filter.js b/plugins/MultiSites/angularjs/dashboard/dashboard-group.filter.js
deleted file mode 100644
index b8f9040e9b..0000000000
--- a/plugins/MultiSites/angularjs/dashboard/dashboard-group.filter.js
+++ /dev/null
@@ -1,67 +0,0 @@
-/*!
- * Piwik - free/libre analytics platform
- *
- * @link http://piwik.org
- * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
- */
-
-/**
- * Filters a given list of websites and groups and makes sure only the websites within a given offset and limit are
- * displayed. It also makes sure sites are displayed under the groups. That means it flattens a structure like this:
- *
- * - website1
- * - website2
- * - website3.sites // this is a group
- *    - website4
- *    - website5
- * - website6
- *
- * to the following structure
- * - website1
- * - website2
- * - website3.sites // this is a group
- * - website4
- * - website5
- * - website6
- */
-(function () {
-    angular.module('piwikApp').filter('multiSitesGroupFilter', multiSitesGroupFilter);
-
-    function multiSitesGroupFilter() {
-        return function(websites, from, to) {
-            var offsetEnd = parseInt(from, 10) + parseInt(to, 10);
-            var groups    = {};
-
-            var sites = [];
-            for (var index = 0; index < websites.length; index++) {
-                var website = websites[index];
-
-                sites.push(website);
-                if (website.sites && website.sites.length) {
-                    groups[website.label] = website;
-                    for (var innerIndex = 0; innerIndex < website.sites.length; innerIndex++) {
-                        sites.push(website.sites[innerIndex]);
-                    }
-                }
-
-                if (sites.length >= offsetEnd) {
-                    break;
-                }
-            }
-
-            // if the first site is a website having a group, then try to find the related group and prepend it to the list
-            // of sites to make sure we always display the name of the group that belongs to a website.
-            var filteredSites = sites.slice(from, offsetEnd);
-
-            if (filteredSites.length && filteredSites[0] && filteredSites[0].group) {
-                var groupName = filteredSites[0].group;
-                if (groups[groupName]) {
-                    filteredSites.unshift(groups[groupName]);
-                }
-            }
-
-            return filteredSites;
-        };
-    }
-})();
-
diff --git a/plugins/MultiSites/angularjs/dashboard/dashboard-model.service.js b/plugins/MultiSites/angularjs/dashboard/dashboard-model.service.js
index f2261aad1c..ee0848ef2e 100644
--- a/plugins/MultiSites/angularjs/dashboard/dashboard-model.service.js
+++ b/plugins/MultiSites/angularjs/dashboard/dashboard-model.service.js
@@ -7,30 +7,14 @@
     multisitesDashboardModel.$inject = ['piwikApi', '$filter', '$timeout'];
 
     function multisitesDashboardModel(piwikApi, $filter, $timeout) {
-        /**
-         *
-         * this is the list of all available sites. For performance reason this list is different to model.sites. ngRepeat
-         * won't operate on the whole list this way. The allSites array contains websites and groups in the following
-         * structure
-         *
-         * - website1
-         * - website2
-         * - website3.sites = [ // this is a group
-         *    - website4
-         *    - website5
-         * ]
-         * - website6
-         *
-         * This structure allows us to display the sites within a group directly under the group without big logic and also
-         * allows us to calculate the summary for each group easily
-         */
-        var allSitesByGroup = [];
+
+        var refreshPromise = null;
 
         // those sites are going to be displayed
         var model = {
             sites        : [],
             isLoading    : false,
-            pageSize     : 5,
+            pageSize     : 25,
             currentPage  : 0,
             totalVisits  : '?',
             totalActions : '?',
@@ -38,6 +22,7 @@
             searchTerm   : '',
             lastVisits   : '?',
             lastVisitsDate : '?',
+            numberOfSites : 0,
             updateWebsitesList: updateWebsitesList,
             getNumberOfFilteredSites: getNumberOfFilteredSites,
             getNumberOfPages: getNumberOfPages,
@@ -46,153 +31,52 @@
             previousPage: previousPage,
             nextPage: nextPage,
             searchSite: searchSite,
-            fetchAllSites: fetchAllSites
+            sortBy: sortBy,
+            reverse: true,
+            sortColumn: 'nb_visits',
+            fetchAllSites: fetchAllSites,
+            refreshInterval: 0
         };
 
-        fetchPreviousSummary();
-
         return model;
 
-        // create a new group object which has similar structure than a website
-        function createGroup(name){
-            return {
-                label: name,
-                sites: [],
-                nb_visits: 0,
-                nb_pageviews: 0,
-                revenue: 0,
-                isGroup: true
-            };
-        }
-
-        // create a new group with empty site to make sure we do not change the original group in $allSites
-        function copyGroup(group)
+        function cancelRefereshInterval()
         {
-            return {
-                label: group.label,
-                sites: [],
-                nb_visits: group.nb_visits,
-                nb_pageviews: group.nb_pageviews,
-                revenue: group.revenue,
-                isGroup: true
+            if (refreshPromise) {
+                $timeout.cancel(refreshPromise);
+                refreshPromise = null;
             };
         }
 
         function onError () {
             model.errorLoadingSites = true;
-            model.sites     = [];
-            allSitesByGroup = [];
-        }
-
-        function calculateMetricsForEachGroup(groups)
-        {
-            angular.forEach(groups, function (group) {
-                angular.forEach(group.sites, function (site) {
-                    var revenue = 0;
-                    if (site.revenue) {
-                        revenue = (site.revenue+'').match(/(\d+\.?\d*)/); // convert $ 0.00 to 0.00 or 5€ to 5
-                    }
-
-                    group.nb_visits    += parseInt(site.nb_visits, 10);
-                    group.nb_pageviews += parseInt(site.nb_pageviews, 10);
-                    if (revenue.length) {
-                        group.revenue += parseInt(revenue[0], 10);
-                    }
-                });
-            });
+            model.sites = [];
         }
 
-        function createGroupsAndMoveSitesIntoRelatedGroup(allSitesUnordered, reportMetadata)
-        {
-            var sitesByGroup = [];
-            var groups = {};
+        function updateWebsitesList(report) {
+            if (!report) {
+                onError();
+                return;
+            }
 
-            // we do 3 things (complete site information, create groups, move sites into group) in one step for
-            // performance reason, there can be > 20k sites
-            angular.forEach(allSitesUnordered, function (site, index) {
-                site.idsite   = reportMetadata[index].idsite;
-                site.group    = reportMetadata[index].group;
-                site.main_url = reportMetadata[index].main_url;
-                // casting evolution to int fixes sorting, see: https://github.com/piwik/piwik/issues/4885
+            var allSites = report.sites;
+            angular.forEach(allSites, function (site, index) {
                 site.visits_evolution    = parseInt(site.visits_evolution, 10);
                 site.pageviews_evolution = parseInt(site.pageviews_evolution, 10);
                 site.revenue_evolution   = parseInt(site.revenue_evolution, 10);
-
-                if (site.group) {
-
-                    if (!groups[site.group]) {
-                        var group = createGroup(site.group);
-
-                        groups[site.group] = group;
-                        sitesByGroup.push(group);
-                    }
-
-                    groups[site.group].sites.push(site);
-
-                } else {
-                    sitesByGroup.push(site);
-                }
             });
 
-            calculateMetricsForEachGroup(groups);
-
-            return sitesByGroup;
-        }
-
-        function getSumTotalActions(allSitesUnordered)
-        {
-            var totalActions = 0;
-
-            if (allSitesUnordered && allSitesUnordered.length) {
-                for (var index in allSitesUnordered) {
-                    var site = allSitesUnordered[index];
-                    if (site && site.nb_pageviews) {
-                        totalActions += parseInt(site.nb_pageviews, 10);
-                    }
-                }
-            }
-
-            return totalActions;
-        }
-
-        function updateWebsitesList(processedReport) {
-            if (!processedReport) {
-                onError();
-                return;
-            }
-
-            var allSitesUnordered = processedReport.reportData;
-
-            model.totalActions = getSumTotalActions(allSitesUnordered);
-            model.totalVisits  = processedReport.reportTotal.nb_visits;
-            model.totalRevenue = processedReport.reportTotal.revenue;
-
-            allSitesByGroup = createGroupsAndMoveSitesIntoRelatedGroup(allSitesUnordered, processedReport.reportMetadata);
-
-            if (!allSitesByGroup.length) {
-                return;
-            }
-
-            if (model.searchTerm) {
-                searchSite(model.searchTerm);
-            } else {
-                model.sites = allSitesByGroup;
-            }
+            model.totalActions  = report.totals.nb_pageviews;
+            model.totalVisits   = report.totals.nb_visits;
+            model.totalRevenue  = report.totals.revenue;
+            model.lastVisits    = report.totals.nb_visits_lastdate;
+            model.sites = allSites;
+            model.numberOfSites  = report.numSites;
+            model.lastVisitsDate = report.lastDate;
         }
 
         function getNumberOfFilteredSites () {
-            var numSites = model.sites.length;
-
-            var groupNames = {};
-
-            for (var index = 0; index < model.sites.length; index++) {
-                var site = model.sites[index];
-                if (site && site.isGroup) {
-                    numSites += site.sites.length;
-                }
-            }
-
-            return numSites;
+            return model.numberOfSites;
         }
 
         function getNumberOfPages() {
@@ -214,100 +98,78 @@
 
         function previousPage() {
             model.currentPage = model.currentPage - 1;
+            fetchAllSites();
         }
 
-        function nextPage() {
-            model.currentPage = model.currentPage + 1;
-        }
-
-        function nestedSearch(sitesByGroup, term)
-        {
-            var filteredSites = [];
+        function sortBy(metric) {
+            if (model.sortColumn == metric) {
+                model.reverse = !model.reverse;
+            }
 
-            term = term.toLowerCase();
+            model.sortColumn = metric;
+            fetchAllSites();
+        };
 
-            for (var index in sitesByGroup) {
-                var site = sitesByGroup[index];
-                if (site.isGroup) {
-                    var matchingSites = nestedSearch(site.sites, term);
-                    if (matchingSites.length || (''+site.label).toLowerCase().indexOf(term) > -1) {
-                        var clonedGroup   = copyGroup(site);
-                        clonedGroup.sites = matchingSites;
-                        filteredSites.push(clonedGroup);
-                    }
-                } else if ((''+site.label).toLowerCase().indexOf(term) > -1) {
-                    filteredSites.push(site);
-                } else if (site.group && (''+site.group).toLowerCase().indexOf(term) > -1) {
-                    filteredSites.push(site);
-                }
-            }
+        function previousPage() {
+            model.currentPage = model.currentPage - 1;
+            fetchAllSites();
+        }
 
-            return filteredSites;
+        function nextPage() {
+            model.currentPage = model.currentPage + 1;
+            fetchAllSites();
         }
 
         function searchSite (term) {
             model.searchTerm  = term;
             model.currentPage = 0;
-            model.sites       = nestedSearch(allSitesByGroup, term);
-        }
-
-        function fetchPreviousSummary () {
-            piwikApi.fetch({
-                method: 'API.getLastDate'
-            }).then(function (response) {
-                if (response && response.value) {
-                    return response.value;
-                }
-            }).then(function (lastDate) {
-                if (!lastDate) {
-                    return;
-                }
-
-                model.lastVisitsDate = lastDate;
-
-                return piwikApi.fetch({
-                    method: 'API.getProcessedReport',
-                    apiModule: 'MultiSites',
-                    apiAction: 'getAll',
-                    hideMetricsDoc: '1',
-                    filter_limit: '0',
-                    showColumns: 'label,nb_visits',
-                    enhanced: 1,
-                    date: lastDate
-                });
-            }).then(function (response) {
-                if (response && response.reportTotal) {
-                    model.lastVisits = response.reportTotal.nb_visits;
-                }
-            });
+            fetchAllSites();
         }
 
-        function fetchAllSites(refreshInterval) {
+        function fetchAllSites() {
 
             if (model.isLoading) {
                 piwikApi.abort();
+                cancelRefereshInterval();
             }
 
             model.isLoading = true;
             model.errorLoadingSites = false;
 
-            return piwikApi.fetch({
-                method: 'API.getProcessedReport',
-                apiModule: 'MultiSites',
-                apiAction: 'getAll',
+            var params = {
+                module: 'MultiSites',
+                action: 'getAllWithGroups',
                 hideMetricsDoc: '1',
-                filter_limit: '-1',
-                showColumns: 'label,nb_visits,nb_pageviews,visits_evolution,pageviews_evolution,revenue_evolution,nb_actions,revenue',
-                enhanced: 1
-            }).then(function (response) {
+                filter_sort_order: 'asc',
+                filter_limit: model.pageSize,
+                filter_offset: getCurrentPagingOffsetStart(),
+                showColumns: 'label,nb_visits,nb_pageviews,visits_evolution,pageviews_evolution,revenue_evolution,nb_actions,revenue'
+            };
+
+            if (model.searchTerm) {
+                params.pattern = model.searchTerm;
+            }
+
+            if (model.sortColumn) {
+                params.filter_sort_column = model.sortColumn;
+            }
+
+            if (model.reverse) {
+                params.filter_sort_order = 'desc';
+            }
+
+            return piwikApi.fetch(params).then(function (response) {
                 updateWebsitesList(response);
             }, onError)['finally'](function () {
                 model.isLoading = false;
 
-                if (refreshInterval && refreshInterval > 0) {
-                    $timeout(function () {
-                        fetchAllSites(refreshInterval);
-                    }, refreshInterval * 1000);
+                if (model.refreshInterval && model.refreshInterval > 0) {
+                    cancelRefereshInterval();
+
+                    refreshPromise = $timeout(function () {
+                        refreshPromise = null;
+                        fetchAllSites(model.refreshInterval);
+                    }, model.refreshInterval * 1000);
                 }
             });
         }
diff --git a/plugins/MultiSites/angularjs/dashboard/dashboard.controller.js b/plugins/MultiSites/angularjs/dashboard/dashboard.controller.js
index d4dd4eba3f..3ba1b15a21 100644
--- a/plugins/MultiSites/angularjs/dashboard/dashboard.controller.js
+++ b/plugins/MultiSites/angularjs/dashboard/dashboard.controller.js
@@ -13,8 +13,6 @@
 
         $scope.model = multisitesDashboardModel;
 
-        $scope.reverse = true;
-        $scope.predicate = 'nb_visits';
         $scope.evolutionSelector = 'visits_evolution';
         $scope.hasSuperUserAccess = piwik.hasSuperUserAccess;
         $scope.date = piwik.broadcast.getValueFromUrl('date');
@@ -22,19 +20,9 @@
         $scope.url  = piwik.piwik_url;
         $scope.period = piwik.period;
 
-        $scope.sortBy = function (metric) {
-
-            var reverse = $scope.reverse;
-            if ($scope.predicate == metric) {
-                reverse = !reverse;
-            }
-
-            $scope.predicate = metric;
-            $scope.reverse   = reverse;
-        };
-
         this.refresh = function (interval) {
-            multisitesDashboardModel.fetchAllSites(interval);
+            multisitesDashboardModel.refreshInterval = interval;
+            multisitesDashboardModel.fetchAllSites();
         };
     }
 })();
diff --git a/plugins/MultiSites/angularjs/dashboard/dashboard.directive.html b/plugins/MultiSites/angularjs/dashboard/dashboard.directive.html
index 287ef25c98..5c9159d076 100644
--- a/plugins/MultiSites/angularjs/dashboard/dashboard.directive.html
+++ b/plugins/MultiSites/angularjs/dashboard/dashboard.directive.html
@@ -12,30 +12,30 @@
     <table id="mt" class="dataTable" cellspacing="0">
         <thead>
         <tr>
-            <th id="names" class="label" ng-click="sortBy('label')" ng-class="{columnSorted: 'label' == predicate}">
+            <th id="names" class="label" ng-click="model.sortBy('label')" ng-class="{columnSorted: 'label' == model.sortColumn}">
                 <span class="heading">{{ 'General_Website'|translate }}</span>
-                <span ng-class="{multisites_asc: !reverse && 'label' == predicate, multisites_desc: reverse && 'label' == predicate}" class="arrow"></span>
+                <span ng-class="{multisites_asc: !model.reverse && 'label' == model.sortColumn, multisites_desc: model.reverse && 'label' == model.sortColumn}" class="arrow"></span>
             </th>
-            <th id="visits" class="multisites-column" ng-click="sortBy('nb_visits')" ng-class="{columnSorted: 'nb_visits' == predicate}">
+            <th id="visits" class="multisites-column" ng-click="model.sortBy('nb_visits')" ng-class="{columnSorted: 'nb_visits' == model.sortColumn}">
                 <span class="heading">{{ 'General_ColumnNbVisits'|translate }}</span>
-                <span ng-class="{multisites_asc: !reverse && 'nb_visits' == predicate, multisites_desc: reverse && 'nb_visits' == predicate}" class="arrow"></span>
+                <span ng-class="{multisites_asc: !model.reverse && 'nb_visits' == model.sortColumn, multisites_desc: model.reverse && 'nb_visits' == model.sortColumn}" class="arrow"></span>
             </th>
-            <th id="pageviews" class="multisites-column" ng-click="sortBy('nb_pageviews')" ng-class="{columnSorted: 'nb_pageviews' == predicate}">
+            <th id="pageviews" class="multisites-column" ng-click="model.sortBy('nb_pageviews')" ng-class="{columnSorted: 'nb_pageviews' == model.sortColumn}">
                 <span class="heading">{{ 'General_ColumnPageviews'|translate }}</span>
-                <span ng-class="{multisites_asc: !reverse && 'nb_pageviews' == predicate, multisites_desc: reverse && 'nb_pageviews' == predicate}" class="arrow"></span>
+                <span ng-class="{multisites_asc: !model.reverse && 'nb_pageviews' == model.sortColumn, multisites_desc: model.reverse && 'nb_pageviews' == model.sortColumn}" class="arrow"></span>
             </th>
 
-            <th ng-if="displayRevenueColumn" id="revenue" class="multisites-column" ng-click="sortBy('revenue')" ng-class="{columnSorted: 'revenue' == predicate}">
+            <th ng-if="displayRevenueColumn" id="revenue" class="multisites-column" ng-click="model.sortBy('revenue')" ng-class="{columnSorted: 'revenue' == model.sortColumn}">
                 <span class="heading">{{ 'General_ColumnRevenue'|translate }}</span>
-                <span ng-class="{multisites_asc: !reverse && 'revenue' == predicate, multisites_desc: reverse && 'revenue' == predicate}" class="arrow"></span>
+                <span ng-class="{multisites_asc: !model.reverse && 'revenue' == model.sortColumn, multisites_desc: model.reverse && 'revenue' == model.sortColumn}" class="arrow"></span>
             </th>
 
-            <th id="evolution" colspan="{{ showSparklines ? 2 : 1 }}" ng-class="{columnSorted: evolutionSelector == predicate}">
-                <span class="arrow" ng-class="{multisites_asc: !reverse && evolutionSelector == predicate, multisites_desc: reverse && evolutionSelector == predicate}"></span>
+            <th id="evolution" colspan="{{ showSparklines ? 2 : 1 }}" ng-class="{columnSorted: evolutionSelector == model.sortColumn}">
+                <span class="arrow" ng-class="{multisites_asc: !model.reverse && evolutionSelector == model.sortColumn, multisites_desc: model.reverse && evolutionSelector == model.sortColumn}"></span>
                 <span class="evolution"
-                      ng-click="sortBy(evolutionSelector)"> {{ 'MultiSites_Evolution'|translate }}</span>
+                      ng-click="model.sortBy(evolutionSelector)"> {{ 'MultiSites_Evolution'|translate }}</span>
                 <select class="selector" id="evolution_selector" ng-model="evolutionSelector"
-                        ng-change="predicate = evolutionSelector">
+                        ng-change="model.sortBy(evolutionSelector)">
                     <option value="visits_evolution">{{ 'General_ColumnNbVisits'|translate }}</option>
                     <option value="pageviews_evolution">{{ 'General_ColumnPageviews'|translate }}</option>
                     <option ng-if="displayRevenueColumn" value="revenue_evolution">{{ 'General_ColumnRevenue'|translate }}</option>
@@ -68,10 +68,10 @@
                 piwik-multisites-site
                 date-sparkline="dateSparkline"
                 show-sparklines="showSparklines"
-                metric="predicate"
+                metric="model.sortColumn"
                 ng-class-odd="'columnodd'"
                 display-revenue-column="displayRevenueColumn"
-                ng-repeat="website in model.sites | orderBy:predicate:reverse | multiSitesGroupFilter:model.getCurrentPagingOffsetStart():model.pageSize">
+                ng-repeat="website in model.sites">
             </tr>
         </tbody>
 
@@ -108,18 +108,13 @@
         <tr row_id="last">
             <td colspan="8" class="site_search">
                 <input type="text"
-                       ng-change="model.searchSite(searchTerm)"
                        ng-model="searchTerm"
+                       piwik-onenter="model.searchSite(searchTerm)"
                        placeholder="{{ 'Actions_SubmenuSitesearch' | translate }}">
-                <img title="Search"
-                     ng-show="!searchTerm"
+                <img title="{{ 'General_ClickToSearch' | translate }}"
+                     ng-click="model.searchSite(searchTerm)"
                      class="search_ico"
                      src="plugins/Morpheus/images/search_ico.png"/>
-                <img title="Clear"
-                     ng-show="searchTerm"
-                     ng-click="searchTerm='';model.searchSite('')"
-                     class="reset"
-                     src="plugins/CoreHome/images/reset_search.png"/>
             </td>
         </tr>
 
diff --git a/plugins/MultiSites/angularjs/dashboard/dashboard.directive.less b/plugins/MultiSites/angularjs/dashboard/dashboard.directive.less
index 6502f4244c..580e3b2772 100644
--- a/plugins/MultiSites/angularjs/dashboard/dashboard.directive.less
+++ b/plugins/MultiSites/angularjs/dashboard/dashboard.directive.less
@@ -86,6 +86,7 @@
     left: -25px;
     margin-right: 0px;
     margin-top: -1px;
+    cursor: pointer;
   }
   .reset {
     position: relative;
diff --git a/plugins/MultiSites/tests/Fixtures/ManySitesWithVisits.php b/plugins/MultiSites/tests/Fixtures/ManySitesWithVisits.php
new file mode 100644
index 0000000000..8305ad9b33
--- /dev/null
+++ b/plugins/MultiSites/tests/Fixtures/ManySitesWithVisits.php
@@ -0,0 +1,83 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link    http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+namespace Piwik\Plugins\MultiSites\tests\Fixtures;
+
+use Piwik\Date;
+use Piwik\Tests\Framework\Fixture;
+
+/**
+ * Generates tracker testing data for our ControllerTest
+ *
+ * This Simple fixture adds one website and tracks one visit with couple pageviews and an ecommerce conversion
+ */
+class ManySitesWithVisits extends Fixture
+{
+    public $dateTime = '2013-01-23 01:23:45';
+    public $idSite = 1;
+
+    public function setUp()
+    {
+        $this->setUpWebsite();
+        $this->trackFirstVisit($this->idSite);
+        $this->trackSecondVisit($this->idSite);
+        $this->trackFirstVisit($siteId = 2);
+        $this->trackSecondVisit($siteId = 3);
+        $this->trackSecondVisit($siteId = 3);
+        $this->trackSecondVisit($siteId = 4);
+    }
+
+    public function tearDown()
+    {
+        // empty
+    }
+
+    private function setUpWebsite()
+    {
+        for ($i = 1; $i <= 15; $i++) {
+            if (!self::siteCreated($i)) {
+                $idSite = self::createWebsite($this->dateTime, $ecommerce = 1, 'Site ' . $i);
+                $this->assertSame($i, $idSite);
+            }
+        }
+    }
+
+    protected function trackFirstVisit($idSite)
+    {
+        $t = self::getTracker($idSite, $this->dateTime, $defaultInit = true);
+
+        $t->setForceVisitDateTime(Date::factory($this->dateTime)->addHour(0.1)->getDatetime());
+        $t->setUrl('http://example.com/');
+        self::checkResponse($t->doTrackPageView('Viewing homepage'));
+
+        $t->setForceVisitDateTime(Date::factory($this->dateTime)->addHour(0.2)->getDatetime());
+        $t->setUrl('http://example.com/sub/page');
+        self::checkResponse($t->doTrackPageView('Second page view'));
+
+        $t->setForceVisitDateTime(Date::factory($this->dateTime)->addHour(0.25)->getDatetime());
+        $t->addEcommerceItem($sku = 'SKU_ID', $name = 'Test item!', $category = 'Test & Category', $price = 777, $quantity = 33);
+        self::checkResponse($t->doTrackEcommerceOrder('TestingOrder', $grandTotal = 33 * 77));
+    }
+
+    protected function trackSecondVisit($idSite)
+    {
+        $t = self::getTracker($idSite, $this->dateTime, $defaultInit = true);
+        $t->setIp('56.11.55.73');
+
+        $t->setForceVisitDateTime(Date::factory($this->dateTime)->addHour(0.1)->getDatetime());
+        $t->setUrl('http://example.com/sub/page');
+        self::checkResponse($t->doTrackPageView('Viewing homepage'));
+
+        $t->setForceVisitDateTime(Date::factory($this->dateTime)->addHour(0.2)->getDatetime());
+        $t->setUrl('http://example.com/?search=this is a site search query');
+        self::checkResponse($t->doTrackPageView('Site search query'));
+
+        $t->setForceVisitDateTime(Date::factory($this->dateTime)->addHour(0.3)->getDatetime());
+        $t->addEcommerceItem($sku = 'SKU_ID2', $name = 'A durable item', $category = 'Best seller', $price = 321);
+        self::checkResponse($t->doTrackEcommerceCartUpdate($grandTotal = 33 * 77));
+    }
+}
\ No newline at end of file
diff --git a/plugins/MultiSites/tests/Integration/ControllerTest.php b/plugins/MultiSites/tests/Integration/ControllerTest.php
new file mode 100644
index 0000000000..cd6ea91b11
--- /dev/null
+++ b/plugins/MultiSites/tests/Integration/ControllerTest.php
@@ -0,0 +1,126 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\MultiSites\tests\Integration;
+
+use Piwik\FrontController;
+use Piwik\Plugins\MultiSites\tests\fixtures\ManySitesWithVisits;
+use Piwik\Tests\Framework\TestCase\SystemTestCase;
+
+/**
+ * @group MultiSites
+ * @group ControllerTest
+ * @group Plugins
+ */
+class ControllerTest extends SystemTestCase
+{
+    /**
+     * @var ManySitesWithVisits
+     */
+    public static $fixture = null; // initialized below class definition
+
+    public function test_getAllWithGroups()
+    {
+        $sites = $this->requestGetAllWithGroups(array('filter_limit' => 20));
+        $this->assertTrue(is_string($sites));
+
+        $sites = json_decode($sites, true);
+
+        // as limit is 20 make sure it returns all 15 sites but we do not check for all the detailed sites info,
+        // this is tested in other tests. We only check for first site.
+        $this->assertSame(15, count($sites['sites']));
+        $this->assertEquals(array(
+            'label' => 'Site 1',
+            'nb_visits' => 2,
+            'nb_pageviews' => 3,
+            'revenue' => '$ 2541',
+            'visits_evolution' => '100%',
+            'pageviews_evolution' => '100%',
+            'revenue_evolution' => '100%',
+            'idsite' => 1,
+            'group' => '',
+            'main_url' => 'http://piwik.net',
+        ), $sites['sites'][0]);
+
+        unset($sites['sites']);
+        $expected = array(
+            'numSites' => 15,
+            'totals' => array(
+                'nb_pageviews' => 8,
+                'nb_visits' => 5,
+                'revenue' => 5082,
+                'nb_visits_lastdate' => 0,
+            ),
+            'lastDate' => '2013-01-22'
+        );
+
+        $this->assertEquals($expected, $sites);
+    }
+
+    public function test_getAllWithGroups_ifLimitIsApplied_ShouldStill_ReturnCorrectNumberOfSitesAvailable()
+    {
+        $sites = $this->requestGetAllWithGroups(array('filter_limit' => 5));
+        $sites = json_decode($sites, true);
+
+        $this->assertSame(5, count($sites['sites']));
+        $this->assertSame(15, $sites['numSites']);
+        $this->assertReturnedSitesEquals(array(1, 2, 3, 4, 5), $sites);
+    }
+
+    public function test_getAllWithGroups_shouldBeAbleToHandleLimitAndOffset()
+    {
+        $sites = $this->requestGetAllWithGroups(array('filter_limit' => 5, 'filter_offset' => 4));
+        $sites = json_decode($sites, true);
+
+        $this->assertSame(5, count($sites['sites']));
+        $this->assertSame(15, $sites['numSites']);
+        $this->assertReturnedSitesEquals(array(5, 6, 7, 8, 9), $sites);
+    }
+
+    public function test_getAllWithGroups_shouldApplySearchAndReturnInNumSitesOnlyTheNumberOfMatchingSites()
+    {
+        $pattern = 'Site 1';
+        $sites = $this->requestGetAllWithGroups(array('filter_limit' => 5, 'pattern' => $pattern));
+        $sites = json_decode($sites, true);
+
+        $this->assertSame(5, count($sites['sites']));
+        $this->assertSame(1 + 6, $sites['numSites']); // Site 1 + Site10-15
+        $this->assertReturnedSitesEquals(array(1, 10, 11, 12, 13), $sites);
+    }
+
+    private function assertReturnedSitesEquals($expectedSiteIds, $sites)
+    {
+        foreach ($expectedSiteIds as $index => $expectedSiteId) {
+            $this->assertSame($expectedSiteId, $sites['sites'][$index]['idsite']);
+        }
+    }
+
+    private function requestGetAllWithGroups($params)
+    {
+        $oldGet = $_GET;
+        $params['period'] = 'day';
+        $params['date']   = '2013-01-23';
+        $_GET   = $params;
+        $sites  = FrontController::getInstance()->dispatch('MultiSites', 'getAllWithGroups');
+        $_GET   = $oldGet;
+        return $sites;
+    }
+
+    public static function getOutputPrefix()
+    {
+        return '';
+    }
+
+    public static function getPathToTestDirectory()
+    {
+        return dirname(__FILE__);
+    }
+
+}
+
+ControllerTest::$fixture = new ManySitesWithVisits();
\ No newline at end of file
diff --git a/plugins/MultiSites/tests/Integration/DashboardTest.php b/plugins/MultiSites/tests/Integration/DashboardTest.php
new file mode 100644
index 0000000000..33d671cbb2
--- /dev/null
+++ b/plugins/MultiSites/tests/Integration/DashboardTest.php
@@ -0,0 +1,401 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\MultiSites\tests\Integration;
+
+use Piwik\DataTable;
+use Piwik\Period;
+use Piwik\Plugins\MultiSites\Dashboard;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+
+/**
+ * @group MultiSites
+ * @group DashboardTest
+ * @group Dashboard
+ * @group Plugins
+ */
+class DashboardTest extends IntegrationTestCase
+{
+    /**
+     * @var Dashboard
+     */
+    private $dashboard;
+
+    private $numSitesToCreate = 3;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        for ($i = 1; $i <= $this->numSitesToCreate; $i++) {
+            Fixture::createWebsite('2012-12-12 00:00:00', $ecommerce = 0, 'Site ' . $i);
+        }
+
+        $this->dashboard = $this->getMockBuilder('Piwik\Plugins\MultiSites\Dashboard')
+                                ->setMethods(null)
+                                ->disableOriginalConstructor()
+                                ->getMock();
+    }
+
+    public function test__construct_shouldFetchSitesWithNeededColumns_AndReturnEvenSitesHavingNoVisits()
+    {
+        $dayToFetch = '2012-12-13';
+        $lastDate   = '2012-12-12';
+
+        $dashboard = new Dashboard('day', $dayToFetch, false);
+
+        $this->assertSame($this->numSitesToCreate, $dashboard->getNumSites());
+        $this->assertEquals($lastDate, $dashboard->getLastDate());
+
+        $expectedTotals = array(
+            'nb_pageviews' => 0,
+            'nb_visits' => 0,
+            'revenue' => 0,
+            'nb_visits_lastdate' => 0,
+        );
+        $this->assertEquals($expectedTotals, $dashboard->getTotals());
+
+        $expectedSites = array (
+            array (
+                'label' => 'Site 1',
+                'nb_visits' => 0,
+                'nb_pageviews' => 0,
+                'revenue' => '$ 0',
+                'visits_evolution' => '0%',
+                'pageviews_evolution' => '0%',
+                'revenue_evolution' => '0%',
+                'idsite' => 1,
+                'group' => '',
+                'main_url' => 'http://piwik.net',
+            ),
+            array (
+                'label' => 'Site 2',
+                'nb_visits' => 0,
+                'nb_pageviews' => 0,
+                'revenue' => '$ 0',
+                'visits_evolution' => '0%',
+                'pageviews_evolution' => '0%',
+                'revenue_evolution' => '0%',
+                'idsite' => 2,
+                'group' => '',
+                'main_url' => 'http://piwik.net',
+            ),
+            array (
+                'label' => 'Site 3',
+                'nb_visits' => 0,
+                'nb_pageviews' => 0,
+                'revenue' => '$ 0',
+                'visits_evolution' => '0%',
+                'pageviews_evolution' => '0%',
+                'revenue_evolution' => '0%',
+                'idsite' => 3,
+                'group' => '',
+                'main_url' => 'http://piwik.net',
+            ),
+        );
+        $this->assertEquals($expectedSites, $dashboard->getSites(array(), $limit = 10));
+    }
+
+    public function test__construct_shouldActuallyFindSitesWhenSeaching()
+    {
+        $dashboard = new Dashboard('day', '2012-12-13', false);
+        $this->assertSame($this->numSitesToCreate, $dashboard->getNumSites());
+
+        $expectedSites = array (
+            array (
+                'label' => 'Site 2',
+                'nb_visits' => 0,
+                'nb_pageviews' => 0,
+                'revenue' => '$ 0',
+                'visits_evolution' => '0%',
+                'pageviews_evolution' => '0%',
+                'revenue_evolution' => '0%',
+                'idsite' => 2,
+                'group' => '',
+                'main_url' => 'http://piwik.net',
+            ),
+        );
+        $dashboard->search('site 2');
+        $this->assertEquals($expectedSites, $dashboard->getSites(array(), $limit = 10));
+        $this->assertSame(1, $dashboard->getNumSites());
+    }
+
+    public function test_getNumSites_shouldBeZeroIfNoSitesAreSet()
+    {
+        $this->assertSame(0, $this->dashboard->getNumSites());
+    }
+
+    public function test_getNumSites_shouldReturnTheNumberOfSetSites()
+    {
+        $this->setSitesTable(4);
+
+        $this->assertSame(4, $this->dashboard->getNumSites());
+    }
+
+    public function test_getSites_shouldReturnAnArrayOfSites()
+    {
+        $this->setSitesTable(8);
+
+        $expectedSites = $this->buildSitesArray(array(1, 2, 3, 4, 5, 6, 7, 8));
+
+        $this->assertEquals($expectedSites, $this->dashboard->getSites(array(), $limit = 20));
+    }
+
+    public function test_getSites_shouldApplyALimit()
+    {
+        $this->setSitesTable(8);
+
+        $expectedSites = $this->buildSitesArray(array(1, 2, 3, 4));
+
+        $this->assertEquals($expectedSites, $this->dashboard->getSites(array(), $limit = 4));
+    }
+
+    public function test_getSites_WithGroup_shouldApplyALimitAndKeepSitesWithinGroup()
+    {
+        $sites = $this->setSitesTable(20);
+
+        $this->setGroupForSiteId($sites, $siteId = 1, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 2, 'group2');
+        $this->setGroupForSiteId($sites, $siteId = 3, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 4, 'group4');
+        $this->setGroupForSiteId($sites, $siteId = 15, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 16, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 18, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 6, 'group4');
+        $this->dashboard->setSitesTable($sites);
+
+        $expectedSites = array (
+            array (
+                'label' => 'group1',
+                'nb_visits' => 50, // there are 5 matching sites having that group, we only return 4, still result is correct!
+                'isGroup' => 1,
+            ), array (
+                'label' => 'Site1',
+                'nb_visits' => 10,
+                'group' => 'group1',
+            ), array (
+                'label' => 'Site3',
+                'nb_visits' => 10,
+                'group' => 'group1',
+            ), array (
+                'label' => 'Site15',
+                'nb_visits' => 10,
+                'group' => 'group1',
+            ),
+        );
+
+        $this->assertEquals($expectedSites, $this->dashboard->getSites(array(), $limit = 4));
+    }
+
+    public function test_search_shouldUpdateTheNumberOfAvailableSites()
+    {
+        $this->setSitesTable(100);
+
+        $this->dashboard->search('site1');
+
+        // site1 + site1* matches
+        $this->assertSame(12, $this->dashboard->getNumSites());
+    }
+
+    public function test_search_shouldOnlyKeepMatchingSites()
+    {
+        $this->setSitesTable(100);
+
+        $this->dashboard->search('site1');
+
+        $expectedSites = $this->buildSitesArray(array(1, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 100));
+
+        $this->assertEquals($expectedSites, $this->dashboard->getSites(array(), $limit = 20));
+    }
+
+    public function test_search_noSiteMatches()
+    {
+        $this->setSitesTable(100);
+
+        $this->dashboard->search('anYString');
+
+        $this->assertSame(0, $this->dashboard->getNumSites());
+        $this->assertEquals(array(), $this->dashboard->getSites(array(), $limit = 20));
+    }
+
+    public function test_search_WithGroup_shouldDoesSearchInGroupNameAndMatchesEvenSitesHavingThatGroupName()
+    {
+        $sites = $this->setSitesTable(20);
+
+        $this->setGroupForSiteId($sites, $siteId = 1, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 2, 'group2');
+        $this->setGroupForSiteId($sites, $siteId = 3, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 4, 'group4');
+        $this->setGroupForSiteId($sites, $siteId = 15, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 16, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 18, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 6, 'group4');
+
+        $this->dashboard->setSitesTable($sites);
+        $this->dashboard->search('group');
+
+        // groups within that site should be listed first.
+        $expectedSites = array (
+            array (
+                'label' => 'group1',
+                'nb_visits' => 50,
+                'isGroup' => 1,
+            ),
+            array (
+                'label' => 'Site1',
+                'nb_visits' => 10,
+                'group' => 'group1',
+            ),
+            array (
+                'label' => 'Site3',
+                'nb_visits' => 10,
+                'group' => 'group1',
+            ),
+            array (
+                'label' => 'Site15',
+                'nb_visits' => 10,
+                'group' => 'group1',
+            ),
+            array (
+                'label' => 'Site16',
+                'nb_visits' => 10,
+                'group' => 'group1',
+            ),
+            array (
+                'label' => 'Site18',
+                'nb_visits' => 10,
+                'group' => 'group1',
+            ),
+            array (
+                'label' => 'group4',
+                'nb_visits' => 20,
+                'isGroup' => 1,
+            ),
+            array (
+                'label' => 'Site4',
+                'nb_visits' => 10,
+                'group' => 'group4',
+            ),
+            array (
+                'label' => 'Site6',
+                'nb_visits' => 10,
+                'group' => 'group4',
+            ),
+            array (
+                'label' => 'group2',
+                'nb_visits' => 10,
+                'isGroup' => 1,
+            ),
+            array (
+                'label' => 'Site2',
+                'nb_visits' => 10,
+                'group' => 'group2',
+            ),
+        );
+
+        // 3 groups + 8 sites having a group.
+        $this->assertSame(3 + 8, $this->dashboard->getNumSites());
+
+        $matchingSites = $this->dashboard->getSites(array(), $limit = 20);
+        $this->assertEquals($expectedSites, $matchingSites);
+
+        // test with limit should only return the first results
+        $matchingSites = $this->dashboard->getSites(array(), $limit = 8);
+        $this->assertEquals(array_slice($expectedSites, 0, 8), $matchingSites);
+    }
+
+    public function test_search_WithGroup_IfASiteMatchesButNotTheGroupName_ItShouldKeepTheGroupThough()
+    {
+        $sites = $this->setSitesTable(20);
+
+        $this->setGroupForSiteId($sites, $siteId = 1, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 2, 'group2');
+        $this->setGroupForSiteId($sites, $siteId = 3, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 20, 'group4');
+        $this->setGroupForSiteId($sites, $siteId = 15, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 16, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 18, 'group1');
+        $this->setGroupForSiteId($sites, $siteId = 6, 'group4');
+
+        $this->dashboard->setSitesTable($sites);
+        $this->dashboard->search('site2');
+
+        $expectedSites = array (
+            array (
+                'label' => 'group4',
+                'nb_visits' => 20, // another site belongs to that group which doesn't match that name yet still we need to sum the correct result.
+                'isGroup' => 1,
+            ),
+            array (
+                'label' => 'Site20',
+                'nb_visits' => 10,
+                'group' => 'group4',
+            ),
+            array (
+                'label' => 'group2',
+                'nb_visits' => 10,
+                'isGroup' => 1,
+            ),
+            array (
+                'label' => 'Site2',
+                'nb_visits' => 10,
+                'group' => 'group2',
+            ),
+        );
+
+        // 2 matching sites + their group
+        $this->assertSame(2 + 2, $this->dashboard->getNumSites());
+
+        $matchingSites = $this->dashboard->getSites(array(), $limit = 20);
+        $this->assertEquals($expectedSites, $matchingSites);
+    }
+
+    public function test_getLastDate_shouldReturnTheLastDate_IfAnyIsSet()
+    {
+        $this->setSitesTable(1);
+
+        $this->assertSame('2012-12-12', $this->dashboard->getLastDate());
+    }
+
+    public function test_getLastDate_shouldReturnAnEmptyString_IfNoLastDateIsSet()
+    {
+        $this->dashboard->setSitesTable(new DataTable());
+
+        $this->assertSame('', $this->dashboard->getLastDate());
+    }
+
+    private function setGroupForSiteId(DataTable $table, $siteId, $groupName)
+    {
+        $table->getRowFromLabel('Site' . $siteId)->setMetadata('group', $groupName);
+    }
+
+    private function setSitesTable($numSites)
+    {
+        $sites = new DataTable();
+        $sites->addRowsFromSimpleArray($this->buildSitesArray(range(1, $numSites)));
+        $sites->setMetadata('last_period_date', Period\Factory::build('day', '2012-12-12'));
+
+        $this->dashboard->setSitesTable($sites);
+
+        return $sites;
+    }
+
+    private function buildSitesArray($siteIds)
+    {
+        $sites = array();
+
+        foreach ($siteIds as $siteId) {
+            $sites[] = array('label' => 'Site' . $siteId, 'nb_visits' => 10);
+        }
+
+        return $sites;
+
+    }
+
+}