Better segment editor and fixes (#12040)

* column tweak

* fix install

* more tweaks

* rename column to dimension

* various fixes

* added new control expandable select

* starting to refactor segment selector

* make segment editor work again

* use translation keys

* defined some metrics

* set types

* simplify

* simplify

* fix join generator

* add possibility to use custom join table names when using query builder and it uses an inner query

* fix bug in query selector when selecting same field name from different tables twice

* more metadata

* more tweaks

* improve selector

* add possibility to use custom entity names

* also processed archived metrics

* generate sql filter, suggested values callback, and accept values automatically for columns with enums

* several tweaks

* focus search field when opening it

* various tweaks

* added missing method

* format and fix more metadata

* more fixes

* better definition

* define custom filter

* fix definition

* fix various tests

* fix more tests

* fix bug in logquery builder

* fix referrerurl segment was missing

* fix some tests

* fix more tests

* add group

* refactor for better definition

* fix a bug in log query builder when similar columns are used in archiver

* add goal metrics

* various fixes

* make datatable row more flexible

* various fixes and visualization enhancements

* simply segment editor and make it smaller

* remove trailing comma

* various fixes and added new dimension

* fix formatting of returning customer

* added missing primary key

* fixes

* various fixes and improvements

* make sure to update segment definition when selecting a value from auto complete list

* various fixes and more metrics

* more metrics

* more dimensions and fixes

* fix some tests

* fix some integration tests

* update submodule

* fix some system tests

* fix ui tests

* trigger new test run

* fix more ui tests

* fix system tests

* update submodule

* fix categories

* sort segments by category for more consistency

* add custom variables

* some translations and fixes

* add minute segment

* more segments

* added plurals

* added some docs

* fix test

* fix tests

* fix tests

* added suggested values

* fix some tests

* various fixes

* fix more tests

* allow to select segments on any site

* make sure to include file

* added doc block

* fix some system tests

* fix most system tests

* fix ui test

* fix system test

* adjust examples

* added more tests and docs

* no metrics for these dimensions

* added developer changelog and made some classes public api

* some fixes for entity names

* add possibility to set format metrics in test

* more consistency in defining the name

* get idsites only if provided

* fix integration tests

* added another segment for visit start hour and visit start minute

* more clear name for segment

* use old segment name to not break bc

* various fixes

* more test fixes

* fix no suggested values for new segment

* add event value

* for boolean dimensions only sum metric

* update available widgets when updating reporting menu

* Add new segments in developer changelog + typo

* fix system tests

* fix screenshot test

4 files changed, 82 insertions, 21 deletions

diff --git a/plugins/API/API.php b/plugins/API/API.php
index 2100ea825f..4e27e5e663 100644
--- a/plugins/API/API.php
+++ b/plugins/API/API.php
@@ -29,6 +29,7 @@ use Piwik\Plugin\SettingsProvider;
 use Piwik\Plugins\API\DataTable\MergeDataTables;
 use Piwik\Plugins\CoreAdminHome\CustomLogo;
 use Piwik\Plugins\CorePluginsAdmin\SettingsMetadata;
+use Piwik\Site;
 use Piwik\Translation\Translator;
 use Piwik\Measurable\Type\TypeManager;
 use Piwik\Version;
@@ -152,6 +153,7 @@ class API extends \Piwik\Plugin\API
         if (empty($idSites)) {
             Piwik::checkUserHasSomeViewAccess();
         } else {
+            $idSites = Site::getIdSitesFromIdSitesString($idSites);
             Piwik::checkUserHasViewAccess($idSites);
         }
 
@@ -433,9 +435,27 @@ class API extends \Piwik\Plugin\API
     {
         Piwik::checkUserHasViewAccess($idSite);
 
+        $apiParameters = array();
+        $entityNames = StaticContainer::get('entities.idNames');
+        foreach ($entityNames as $entityName) {
+            if ($entityName === 'idGoal' && $idGoal) {
+                $apiParameters['idGoal'] = $idGoal;
+            } elseif ($entityName === 'idDimension' && $idDimension) {
+                $apiParameters['idDimension'] = $idDimension;
+            } else {
+                // ideally it would get the value from API params but dynamic params is not possible yet in API. If this
+                // method is called eg in Request::processRequest, it could in theory pick up a param from the original request
+                // and not from the API request within the original request.
+                $idEntity = Common::getRequestVar($entityName, 0, 'int');
+                if ($idEntity > 0) {
+                    $apiParameters[$entityName] = $idEntity;
+                }
+            }
+        }
+
         $rowEvolution = new RowEvolution();
         return $rowEvolution->getRowEvolution($idSite, $period, $date, $apiModule, $apiAction, $label, $segment, $column,
-            $language, $idGoal, $legendAppendMetric, $labelUseAbsoluteUrl, $idDimension);
+            $language, $apiParameters, $legendAppendMetric, $labelUseAbsoluteUrl);
     }
 
     /**
diff --git a/plugins/API/ProcessedReport.php b/plugins/API/ProcessedReport.php
index 360d6b7ddf..1cd327c8c7 100644
--- a/plugins/API/ProcessedReport.php
+++ b/plugins/API/ProcessedReport.php
@@ -14,6 +14,7 @@ use Piwik\Archive\DataTableFactory;
 use Piwik\CacheId;
 use Piwik\Cache as PiwikCache;
 use Piwik\Common;
+use Piwik\Container\StaticContainer;
 use Piwik\DataTable;
 use Piwik\DataTable\Row;
 use Piwik\DataTable\Simple;
@@ -48,6 +49,18 @@ class ProcessedReport
     {
         $reportsMetadata = $this->getReportMetadata($idSite, $period, $date, $hideMetricsDoc, $showSubtableReports);
 
+        $entityNames = StaticContainer::get('entities.idNames');
+        foreach ($entityNames as $entityName) {
+            if ($entityName === 'idGoal' || $entityName === 'idDimension') {
+                continue; // idGoal and idDimension is passed directly but for other entities we need to "workaround" and
+                // check for eg idFoo from GET/POST because we cannot add parameters to API dynamically
+            }
+            $idEntity = Common::getRequestVar($entityName, 0, 'int');
+            if ($idEntity > 0) {
+                $apiParameters[$entityName] = $idEntity;
+            }
+        }
+
         foreach ($reportsMetadata as $report) {
             // See ArchiveProcessor/Aggregator.php - unique visitors are not processed for period != day
             // todo: should use SettingsPiwik::isUniqueVisitorsEnabled instead
@@ -73,6 +86,7 @@ class ProcessedReport
                 }
             }
         }
+
         return false;
     }
 
diff --git a/plugins/API/RowEvolution.php b/plugins/API/RowEvolution.php
index f7af6b93b1..41c75bffde 100644
--- a/plugins/API/RowEvolution.php
+++ b/plugins/API/RowEvolution.php
@@ -36,7 +36,7 @@ class RowEvolution
         'getPageUrl'
     );
 
-    public function getRowEvolution($idSite, $period, $date, $apiModule, $apiAction, $label = false, $segment = false, $column = false, $language = false, $idGoal = false, $legendAppendMetric = true, $labelUseAbsoluteUrl = true, $idDimension = false)
+    public function getRowEvolution($idSite, $period, $date, $apiModule, $apiAction, $label = false, $segment = false, $column = false, $language = false, $apiParameters = array(), $legendAppendMetric = true, $labelUseAbsoluteUrl = true)
     {
         // validation of requested $period & $date
         if ($period == 'range') {
@@ -51,9 +51,9 @@ class RowEvolution
         $label = DataTablePostProcessor::unsanitizeLabelParameter($label);
         $labels = Piwik::getArrayFromApiParameter($label);
 
-        $metadata = $this->getRowEvolutionMetaData($idSite, $period, $date, $apiModule, $apiAction, $language, $idGoal, $idDimension);
+        $metadata = $this->getRowEvolutionMetaData($idSite, $period, $date, $apiModule, $apiAction, $language, $apiParameters);
 
-        $dataTable = $this->loadRowEvolutionDataFromAPI($metadata, $idSite, $period, $date, $apiModule, $apiAction, $labels, $segment, $idGoal, $idDimension);
+        $dataTable = $this->loadRowEvolutionDataFromAPI($metadata, $idSite, $period, $date, $apiModule, $apiAction, $labels, $segment, $apiParameters);
 
         if (empty($labels)) {
             $labels = $this->getLabelsFromDataTable($dataTable, $labels);
@@ -244,11 +244,11 @@ class RowEvolution
      * @param string $apiAction
      * @param string|bool $label
      * @param string|bool $segment
-     * @param int|bool $idGoal
+     * @param array $apiParameters
      * @throws Exception
      * @return DataTable\Map|DataTable
      */
-    private function loadRowEvolutionDataFromAPI($metadata, $idSite, $period, $date, $apiModule, $apiAction, $label = false, $segment = false, $idGoal = false, $idDimension = false)
+    private function loadRowEvolutionDataFromAPI($metadata, $idSite, $period, $date, $apiModule, $apiAction, $label = false, $segment = false, $apiParameters)
     {
         if (!is_array($label)) {
             $label = array($label);
@@ -264,9 +264,6 @@ class RowEvolution
             'format'                   => 'original',
             'serialize'                => '0',
             'segment'                  => $segment,
-            'idGoal'                   => $idGoal,
-            'idDimension'              => $idDimension,
-
             // data for row evolution should NOT be limited
             'filter_limit'             => -1,
 
@@ -275,6 +272,11 @@ class RowEvolution
             // can be sorted in a different order)
             'labelFilterAddLabelIndex' => count($label) > 1 ? 1 : 0,
         );
+        if (!empty($apiParameters) && is_array($apiParameters)) {
+            foreach ($apiParameters as $param => $value) {
+                $parameters[$param] = $value;
+            }
+        }
 
         // add "processed metrics" like actions per visit or bounce rate
         // note: some reports should not be filtered with AddColumnProcessedMetrics
@@ -306,19 +308,12 @@ class RowEvolution
      * @param $apiModule
      * @param $apiAction
      * @param $language
-     * @param $idGoal
+     * @param $apiParameters
      * @throws Exception
      * @return array
      */
-    private function getRowEvolutionMetaData($idSite, $period, $date, $apiModule, $apiAction, $language, $idGoal = false, $idDimension = false)
+    private function getRowEvolutionMetaData($idSite, $period, $date, $apiModule, $apiAction, $language, $apiParameters)
     {
-        $apiParameters = array();
-        if (!empty($idGoal) && $idGoal > 0) {
-            $apiParameters = array('idGoal' => $idGoal);
-        }
-        if (!empty($idDimension) && $idDimension > 0) {
-            $apiParameters = array('idDimension' => (int) $idDimension);
-        }
         $reportMetadata = API::getInstance()->getMetadata($idSite, $apiModule, $apiAction, $apiParameters, $language,
             $period, $date, $hideMetricsDoc = false, $showSubtableReports = true);
 
diff --git a/plugins/API/SegmentMetadata.php b/plugins/API/SegmentMetadata.php
index 1cce5fd3dd..03f6f64336 100644
--- a/plugins/API/SegmentMetadata.php
+++ b/plugins/API/SegmentMetadata.php
@@ -8,12 +8,19 @@
  */
 namespace Piwik\Plugins\API;
 
+use Piwik\Category\CategoryList;
 use Piwik\Columns\Dimension;
 use Piwik\Piwik;
 use Piwik\Plugin\Segment;
 
 class SegmentMetadata
 {
+    /**
+     * Map of category name to order
+     * @var array
+     */
+    private $categoryOrder = array();
+
     public function getSegmentsMetadata($idSites = array(), $_hideImplementationData = true, $isAuthenticatedWithViewAccess)
     {
         $segments = array();
@@ -55,9 +62,21 @@ class SegmentMetadata
             $segments[] = $segment->toArray();
         }
 
+        $categoryList = CategoryList::get();
+
         foreach ($segments as &$segment) {
+            $categoryId = $segment['category'];
             $segment['name'] = Piwik::translate($segment['name']);
-            $segment['category'] = Piwik::translate($segment['category']);
+            $segment['category'] = Piwik::translate($categoryId);
+
+            if (!isset($this->categoryOrder[$segment['category']])) {
+                $category = $categoryList->getCategory($categoryId);
+                if (!empty($category)) {
+                    $this->categoryOrder[$segment['category']] = $category->getOrder();
+                } else {
+                    $this->categoryOrder[$segment['category']] = 999;
+                }
+            }
 
             if ($_hideImplementationData) {
                 unset($segment['sqlFilter']);
@@ -81,12 +100,25 @@ class SegmentMetadata
     {
         $customVarCategory = Piwik::translate('CustomVariables_CustomVariables');
 
-        $columns = array('type', 'category', 'name', 'segment');
+        $columns = array('category', 'type', 'name', 'segment');
 
         foreach ($columns as $column) {
             // Keep segments ordered alphabetically inside categories..
             $type = -1;
-            if ($column == 'name') $type = 1;
+            if ($column == 'name') {
+                $type = 1;
+            }
+
+            if ($column === 'category') {
+                $idOrder1 = $this->categoryOrder[$row1[$column]];
+                $idOrder2 = $this->categoryOrder[$row2[$column]];
+
+                if ($idOrder1 === $idOrder2) {
+                    continue;
+                }
+
+                return $idOrder1 > $idOrder2 ? 1 : -1;
+            }
 
             $compare = $type * strcmp($row1[$column], $row2[$column]);