Better segment editor and fixes (#12040)

* column tweak

* fix install

* more tweaks

* rename column to dimension

* various fixes

* added new control expandable select

* starting to refactor segment selector

* make segment editor work again

* use translation keys

* defined some metrics

* set types

* simplify

* simplify

* fix join generator

* add possibility to use custom join table names when using query builder and it uses an inner query

* fix bug in query selector when selecting same field name from different tables twice

* more metadata

* more tweaks

* improve selector

* add possibility to use custom entity names

* also processed archived metrics

* generate sql filter, suggested values callback, and accept values automatically for columns with enums

* several tweaks

* focus search field when opening it

* various tweaks

* added missing method

* format and fix more metadata

* more fixes

* better definition

* define custom filter

* fix definition

* fix various tests

* fix more tests

* fix bug in logquery builder

* fix referrerurl segment was missing

* fix some tests

* fix more tests

* add group

* refactor for better definition

* fix a bug in log query builder when similar columns are used in archiver

* add goal metrics

* various fixes

* make datatable row more flexible

* various fixes and visualization enhancements

* simply segment editor and make it smaller

* remove trailing comma

* various fixes and added new dimension

* fix formatting of returning customer

* added missing primary key

* fixes

* various fixes and improvements

* make sure to update segment definition when selecting a value from auto complete list

* various fixes and more metrics

* more metrics

* more dimensions and fixes

* fix some tests

* fix some integration tests

* update submodule

* fix some system tests

* fix ui tests

* trigger new test run

* fix more ui tests

* fix system tests

* update submodule

* fix categories

* sort segments by category for more consistency

* add custom variables

* some translations and fixes

* add minute segment

* more segments

* added plurals

* added some docs

* fix test

* fix tests

* fix tests

* added suggested values

* fix some tests

* various fixes

* fix more tests

* allow to select segments on any site

* make sure to include file

* added doc block

* fix some system tests

* fix most system tests

* fix ui test

* fix system test

* adjust examples

* added more tests and docs

* no metrics for these dimensions

* added developer changelog and made some classes public api

* some fixes for entity names

* add possibility to set format metrics in test

* more consistency in defining the name

* get idsites only if provided

* fix integration tests

* added another segment for visit start hour and visit start minute

* more clear name for segment

* use old segment name to not break bc

* various fixes

* more test fixes

* fix no suggested values for new segment

* add event value

* for boolean dimensions only sum metric

* update available widgets when updating reporting menu

* Add new segments in developer changelog + typo

* fix system tests

* fix screenshot test

7 files changed, 148 insertions, 1 deletions

diff --git a/plugins/CustomVariables/Categories/CustomVariablesCategory.php b/plugins/CustomVariables/Categories/CustomVariablesCategory.php
new file mode 100644
index 0000000000..ced9de0563
--- /dev/null
+++ b/plugins/CustomVariables/Categories/CustomVariablesCategory.php
@@ -0,0 +1,18 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\CustomVariables\Categories;
+
+use Piwik\Category\Category;
+
+class CustomVariablesCategory extends Category
+{
+    protected $id = 'CustomVariables_CustomVariables';
+    protected $order = 8;
+
+}
diff --git a/plugins/CustomVariables/Columns/SearchCategory.php b/plugins/CustomVariables/Columns/SearchCategory.php
new file mode 100644
index 0000000000..abeac9f865
--- /dev/null
+++ b/plugins/CustomVariables/Columns/SearchCategory.php
@@ -0,0 +1,26 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\CustomVariables\Columns;
+
+use Piwik\Columns\Discriminator;
+use Piwik\Plugin\Dimension\ActionDimension;
+use Piwik\Plugins\Actions\Actions\ActionSiteSearch;
+
+class SearchCategory extends ActionDimension
+{
+    protected $type = self::TYPE_TEXT;
+    protected $columnName = 'custom_var_v4';
+    protected $nameSingular = 'Actions_SiteSearchCategory';
+    protected $namePlural = 'Actions_SiteSearchCategories';
+
+    public function getDbDiscriminator()
+    {
+        return new Discriminator($this->dbTableName, 'custom_var_k4', ActionSiteSearch::CVAR_KEY_SEARCH_CATEGORY);
+    }
+}
\ No newline at end of file
diff --git a/plugins/CustomVariables/CustomDimension.php b/plugins/CustomVariables/CustomDimension.php
new file mode 100644
index 0000000000..cd899503cf
--- /dev/null
+++ b/plugins/CustomVariables/CustomDimension.php
@@ -0,0 +1,36 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\CustomVariables;
+
+use Piwik\Columns\Dimension;
+use Piwik\Piwik;
+
+class CustomDimension extends Dimension
+{
+    protected $type = self::TYPE_TEXT;
+
+    private $id;
+
+    public function getId()
+    {
+        return $this->id;
+    }
+
+    public function initCustomDimension($index, Model $scope)
+    {
+        $category = $scope->getScopeDescription();
+
+        $this->id = 'CustomVariables.CustomVariable' . ucfirst($scope->getScope()) . $index;
+        $this->nameSingular = Piwik::translate('CustomVariables_ColumnCustomVariableValue') . ' ' . $index . ' (' . $category .')';
+        $this->columnName = 'custom_var_v' . (int) $index;
+        $this->category = 'CustomVariables_CustomVariables';
+        $this->dbTableName = $scope->getUnprefixedTableName();
+    }
+
+}
\ No newline at end of file
diff --git a/plugins/CustomVariables/CustomVariables.php b/plugins/CustomVariables/CustomVariables.php
index e5a13bc52a..c18eaf946c 100644
--- a/plugins/CustomVariables/CustomVariables.php
+++ b/plugins/CustomVariables/CustomVariables.php
@@ -23,6 +23,7 @@ class CustomVariables extends \Piwik\Plugin
             'AssetManager.getJavaScriptFiles' => 'getJsFiles',
             'Translate.getClientSideTranslationKeys' => 'getClientSideTranslationKeys',
             'AssetManager.getStylesheetFiles'  => 'getStylesheetFiles',
+            'Dimension.addDimensions' => 'addDimensions',
             'Actions.getCustomActionDimensionFieldsAndJoins' => 'provideActionDimensionFields'
         );
     }
@@ -37,6 +38,24 @@ class CustomVariables extends \Piwik\Plugin
         Model::uninstall();
     }
 
+    public function addDimensions(&$instances)
+    {
+        foreach (Model::getScopes() as $scope) {
+            $model = new Model($scope);
+            try {
+                $highestIndex = $model->getHighestCustomVarIndex();
+            } catch (\Exception $e) {
+                continue; // ignore error for tests to work as this might be executed before Piwik tables are installed
+            }
+
+            foreach (range(1, $highestIndex) as $index) {
+                $custom = new CustomDimension();
+                $custom->initCustomDimension($index, $model);
+                $instances[] = $custom;
+            }
+        }
+    }
+
     /**
      * There are also some hardcoded places in JavaScript
      * @return int
diff --git a/plugins/CustomVariables/Model.php b/plugins/CustomVariables/Model.php
index 235809e1da..5abb327f0e 100644
--- a/plugins/CustomVariables/Model.php
+++ b/plugins/CustomVariables/Model.php
@@ -12,6 +12,7 @@ use Piwik\Common;
 use Piwik\DataTable;
 use Piwik\Db;
 use Piwik\Log;
+use Piwik\Piwik;
 
 class Model
 {
@@ -23,6 +24,7 @@ class Model
 
     private $scope = null;
     private $table = null;
+    private $tableUnprefixed = null;
 
     public function __construct($scope)
     {
@@ -31,7 +33,13 @@ class Model
         }
 
         $this->scope = $scope;
-        $this->table = Common::prefixTable($this->getTableNameFromScope($scope));
+        $this->tableUnprefixed = $this->getTableNameFromScope($scope);
+        $this->table = Common::prefixTable($this->tableUnprefixed);
+    }
+
+    public function getUnprefixedTableName()
+    {
+        return $this->tableUnprefixed;
     }
 
     private function getTableNameFromScope($scope)
@@ -47,11 +55,30 @@ class Model
         }
     }
 
+    public function getScope()
+    {
+        return $this->scope;
+    }
+
     public function getScopeName()
     {
         return ucfirst($this->scope);
     }
 
+    public function getScopeDescription()
+    {
+        switch ($this->scope) {
+            case Model::SCOPE_PAGE:
+                return Piwik::translate('CustomVariables_ScopePage');
+            case Model::SCOPE_VISIT:
+                return Piwik::translate('CustomVariables_ScopeVisit');
+            case Model::SCOPE_CONVERSION:
+                return Piwik::translate('CustomVariables_ScopeConversion');
+        }
+
+        return ucfirst($this->scope);
+    }
+
     /**
      * @see getHighestCustomVarIndex()
      * @return int
diff --git a/plugins/CustomVariables/lang/en.json b/plugins/CustomVariables/lang/en.json
index e065886759..56a6d3380b 100644
--- a/plugins/CustomVariables/lang/en.json
+++ b/plugins/CustomVariables/lang/en.json
@@ -7,6 +7,7 @@
         "PluginDescription": "Custom Variables are (name, value) pairs that you can assign using the Javascript API to visitors or any of their action. Piwik will then report how many visits, pages, conversions for each of these custom names and values. View the detailed Custom Variables for each user and action in the Visitor Log.<br />Required to use <a href=\"http://piwik.org/docs/ecommerce-analytics/\">Ecommerce Analytics</a> feature!",
         "ScopePage": "scope page",
         "ScopeVisit": "scope visit",
+        "ScopeConversion": "scope conversion",
         "ManageDescription": "This overview shows all custom variable slots and their usages for website '%s'. The names within each slot are ordered by how often they were used in total.",
         "ScopeX": "Scope %s",
         "Index": "Index",
diff --git a/plugins/CustomVariables/tests/Integration/ModelTest.php b/plugins/CustomVariables/tests/Integration/ModelTest.php
index 905a5f6a1d..6f504f9525 100644
--- a/plugins/CustomVariables/tests/Integration/ModelTest.php
+++ b/plugins/CustomVariables/tests/Integration/ModelTest.php
@@ -102,6 +102,26 @@ class ModelTest extends IntegrationTestCase
         $this->assertEquals('Conversion', $this->getConversionScope()->getScopeName());
     }
 
+    public function testGetScopeDescription()
+    {
+        $this->assertEquals('scope page', $this->getPageScope()->getScopeDescription());
+        $this->assertEquals('scope visit', $this->getVisitScope()->getScopeDescription());
+        $this->assertEquals('scope conversion', $this->getConversionScope()->getScopeDescription());
+    }
+
+    public function testGetUnprefixedTableName()
+    {
+        $this->assertEquals('log_link_visit_action', $this->getPageScope()->getUnprefixedTableName());
+        $this->assertEquals('log_visit', $this->getVisitScope()->getUnprefixedTableName());
+        $this->assertEquals('log_conversion', $this->getConversionScope()->getUnprefixedTableName());
+    }
+
+    public function testGetScope()
+    {
+        $this->assertEquals(Model::SCOPE_PAGE, $this->getPageScope()->getScope());
+        $this->assertEquals(Model::SCOPE_VISIT, $this->getVisitScope()->getScope());
+    }
+
     public function test_getCurrentNumCustomVars()
     {
         $this->assertEquals(5, $this->getPageScope()->getCurrentNumCustomVars());