Better segment editor and fixes (#12040)

* column tweak

* fix install

* more tweaks

* rename column to dimension

* various fixes

* added new control expandable select

* starting to refactor segment selector

* make segment editor work again

* use translation keys

* defined some metrics

* set types

* simplify

* simplify

* fix join generator

* add possibility to use custom join table names when using query builder and it uses an inner query

* fix bug in query selector when selecting same field name from different tables twice

* more metadata

* more tweaks

* improve selector

* add possibility to use custom entity names

* also processed archived metrics

* generate sql filter, suggested values callback, and accept values automatically for columns with enums

* several tweaks

* focus search field when opening it

* various tweaks

* added missing method

* format and fix more metadata

* more fixes

* better definition

* define custom filter

* fix definition

* fix various tests

* fix more tests

* fix bug in logquery builder

* fix referrerurl segment was missing

* fix some tests

* fix more tests

* add group

* refactor for better definition

* fix a bug in log query builder when similar columns are used in archiver

* add goal metrics

* various fixes

* make datatable row more flexible

* various fixes and visualization enhancements

* simply segment editor and make it smaller

* remove trailing comma

* various fixes and added new dimension

* fix formatting of returning customer

* added missing primary key

* fixes

* various fixes and improvements

* make sure to update segment definition when selecting a value from auto complete list

* various fixes and more metrics

* more metrics

* more dimensions and fixes

* fix some tests

* fix some integration tests

* update submodule

* fix some system tests

* fix ui tests

* trigger new test run

* fix more ui tests

* fix system tests

* update submodule

* fix categories

* sort segments by category for more consistency

* add custom variables

* some translations and fixes

* add minute segment

* more segments

* added plurals

* added some docs

* fix test

* fix tests

* fix tests

* added suggested values

* fix some tests

* various fixes

* fix more tests

* allow to select segments on any site

* make sure to include file

* added doc block

* fix some system tests

* fix most system tests

* fix ui test

* fix system test

* adjust examples

* added more tests and docs

* no metrics for these dimensions

* added developer changelog and made some classes public api

* some fixes for entity names

* add possibility to set format metrics in test

* more consistency in defining the name

* get idsites only if provided

* fix integration tests

* added another segment for visit start hour and visit start minute

* more clear name for segment

* use old segment name to not break bc

* various fixes

* more test fixes

* fix no suggested values for new segment

* add event value

* for boolean dimensions only sum metric

* update available widgets when updating reporting menu

* Add new segments in developer changelog + typo

* fix system tests

* fix screenshot test

3 files changed, 13 insertions, 295 deletions

diff --git a/core/Plugin/Dimension/ActionDimension.php b/core/Plugin/Dimension/ActionDimension.php
index ce8a1eeda2..4ed4867448 100644
--- a/core/Plugin/Dimension/ActionDimension.php
+++ b/core/Plugin/Dimension/ActionDimension.php
@@ -12,10 +12,7 @@ use Piwik\CacheId;
 use Piwik\Cache as PiwikCache;
 use Piwik\Columns\Dimension;
 use Piwik\Plugin\Manager as PluginManager;
-use Piwik\Plugin\Segment;
-use Piwik\Common;
 use Piwik\Plugin;
-use Piwik\Db;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
@@ -38,111 +35,8 @@ abstract class ActionDimension extends Dimension
 {
     const INSTALLER_PREFIX = 'log_link_visit_action.';
 
-    private $tableName = 'log_link_visit_action';
-
-    /**
-     * Installs the action dimension in case it is not installed yet. The installation is already implemented based on
-     * the {@link $columnName} and {@link $columnType}. If you want to perform additional actions beside adding the
-     * column to the database - for instance adding an index - you can overwrite this method. We recommend to call
-     * this parent method to get the minimum required actions and then add further custom actions since this makes sure
-     * the column will be installed correctly. We also recommend to change the default install behavior only if really
-     * needed. FYI: We do not directly execute those alter table statements here as we group them together with several
-     * other alter table statements do execute those changes in one step which results in a faster installation. The
-     * column will be added to the `log_link_visit_action` MySQL table.
-     *
-     * Example:
-     * ```
-    public function install()
-    {
-        $changes = parent::install();
-        $changes['log_link_visit_action'][] = "ADD INDEX index_idsite_servertime ( idsite, server_time )";
-
-        return $changes;
-    }
-    ```
-     *
-     * @return array An array containing the table name as key and an array of MySQL alter table statements that should
-     *               be executed on the given table. Example:
-     * ```
-    array(
-        'log_link_visit_action' => array("ADD COLUMN `$this->columnName` $this->columnType", "ADD INDEX ...")
-    );
-    ```
-     * @api
-     */
-    public function install()
-    {
-        if (empty($this->columnName) || empty($this->columnType)) {
-            return array();
-        }
-
-        return array(
-            $this->tableName => array("ADD COLUMN `$this->columnName` $this->columnType")
-        );
-    }
-
-    /**
-     * Updates the action dimension in case the {@link $columnType} has changed. The update is already implemented based
-     * on the {@link $columnName} and {@link $columnType}. This method is intended not to overwritten by plugin
-     * developers as it is only supposed to make sure the column has the correct type. Adding additional custom "alter
-     * table" actions would not really work since they would be executed with every {@link $columnType} change. So
-     * adding an index here would be executed whenever the columnType changes resulting in an error if the index already
-     * exists. If an index needs to be added after the first version is released a plugin update class should be
-     * created since this makes sure it is only executed once.
-     *
-     * @return array An array containing the table name as key and an array of MySQL alter table statements that should
-     *               be executed on the given table. Example:
-     * ```
-    array(
-    'log_link_visit_action' => array("MODIFY COLUMN `$this->columnName` $this->columnType", "DROP COLUMN ...")
-    );
-    ```
-     * @ignore
-     */
-    public function update()
-    {
-        if (empty($this->columnName) || empty($this->columnType)) {
-            return array();
-        }
-
-        return array(
-            $this->tableName => array("MODIFY COLUMN `$this->columnName` $this->columnType")
-        );
-    }
-
-    /**
-     * Uninstalls the dimension if a {@link $columnName} and {@link columnType} is set. In case you perform any custom
-     * actions during {@link install()} - for instance adding an index - you should make sure to undo those actions by
-     * overwriting this method. Make sure to call this parent method to make sure the uninstallation of the column
-     * will be done.
-     * @throws Exception
-     * @api
-     */
-    public function uninstall()
-    {
-        if (empty($this->columnName) || empty($this->columnType)) {
-            return;
-        }
-
-        try {
-            $sql = "ALTER TABLE `" . Common::prefixTable($this->tableName) . "` DROP COLUMN `$this->columnName`";
-            Db::exec($sql);
-        } catch (Exception $e) {
-            if (!Db::get()->isErrNo($e, '1091')) {
-                throw $e;
-            }
-        }
-    }
-
-    /**
-     * Get the version of the dimension which is used for update checks.
-     * @return string
-     * @ignore
-     */
-    public function getVersion()
-    {
-        return $this->columnType;
-    }
+    protected $dbTableName = 'log_link_visit_action';
+    protected $category = 'General_Actions';
 
     /**
      * If the value you want to save for your dimension is something like a page title or page url, you usually do not
@@ -192,23 +86,6 @@ abstract class ActionDimension extends Dimension
     }
 
     /**
-     * Adds a new segment. It automatically sets the SQL segment depending on the column name in case none is set
-     * already.
-     * @see \Piwik\Columns\Dimension::addSegment()
-     * @param Segment $segment
-     * @api
-     */
-    protected function addSegment(Segment $segment)
-    {
-        $sqlSegment = $segment->getSqlSegment();
-        if (!empty($this->columnName) && empty($sqlSegment)) {
-            $segment->setSqlSegment($this->tableName . '.' . $this->columnName);
-        }
-
-        parent::addSegment($segment);
-    }
-
-    /**
      * Get all action dimensions that are defined by all activated plugins.
      * @return ActionDimension[]
      * @ignore
diff --git a/core/Plugin/Dimension/ConversionDimension.php b/core/Plugin/Dimension/ConversionDimension.php
index ff1bff55ae..14a242d24f 100644
--- a/core/Plugin/Dimension/ConversionDimension.php
+++ b/core/Plugin/Dimension/ConversionDimension.php
@@ -12,15 +12,11 @@ use Piwik\CacheId;
 use Piwik\Cache as PiwikCache;
 use Piwik\Columns\Dimension;
 use Piwik\Plugin\Manager as PluginManager;
-use Piwik\Common;
-use Piwik\Db;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\GoalManager;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
-use Piwik\Plugin\Segment;
 use Piwik\Plugin;
-use Exception;
 
 /**
  * Defines a new conversion dimension that records any visit related information during tracking.
@@ -40,116 +36,8 @@ abstract class ConversionDimension extends Dimension
 {
     const INSTALLER_PREFIX = 'log_conversion.';
 
-    private $tableName = 'log_conversion';
-
-    /**
-     * Installs the conversion dimension in case it is not installed yet. The installation is already implemented based
-     * on the {@link $columnName} and {@link $columnType}. If you want to perform additional actions beside adding the
-     * column to the database - for instance adding an index - you can overwrite this method. We recommend to call
-     * this parent method to get the minimum required actions and then add further custom actions since this makes sure
-     * the column will be installed correctly. We also recommend to change the default install behavior only if really
-     * needed. FYI: We do not directly execute those alter table statements here as we group them together with several
-     * other alter table statements do execute those changes in one step which results in a faster installation. The
-     * column will be added to the `log_conversion` MySQL table.
-     *
-     * Example:
-     * ```
-    public function install()
-    {
-    $changes = parent::install();
-    $changes['log_conversion'][] = "ADD INDEX index_idsite_servertime ( idsite, server_time )";
-
-    return $changes;
-    }
-    ```
-     *
-     * @return array An array containing the table name as key and an array of MySQL alter table statements that should
-     *               be executed on the given table. Example:
-     * ```
-    array(
-    'log_conversion' => array("ADD COLUMN `$this->columnName` $this->columnType", "ADD INDEX ...")
-    );
-    ```
-     * @api
-     */
-    public function install()
-    {
-        if (empty($this->columnName) || empty($this->columnType)) {
-            return array();
-        }
-
-        return array(
-            $this->tableName => array("ADD COLUMN `$this->columnName` $this->columnType")
-        );
-    }
-
-    /**
-     * @see ActionDimension::update()
-     * @return array
-     * @ignore
-     */
-    public function update()
-    {
-        if (empty($this->columnName) || empty($this->columnType)) {
-            return array();
-        }
-
-        return array(
-            $this->tableName => array("MODIFY COLUMN `$this->columnName` $this->columnType")
-        );
-    }
-
-    /**
-     * Uninstalls the dimension if a {@link $columnName} and {@link columnType} is set. In case you perform any custom
-     * actions during {@link install()} - for instance adding an index - you should make sure to undo those actions by
-     * overwriting this method. Make sure to call this parent method to make sure the uninstallation of the column
-     * will be done.
-     * @throws Exception
-     * @api
-     */
-    public function uninstall()
-    {
-        if (empty($this->columnName) || empty($this->columnType)) {
-            return;
-        }
-
-        try {
-            $sql = "ALTER TABLE `" . Common::prefixTable($this->tableName) . "` DROP COLUMN `$this->columnName`";
-            Db::exec($sql);
-        } catch (Exception $e) {
-            if (!Db::get()->isErrNo($e, '1091')) {
-                throw $e;
-            }
-        }
-    }
-
-    /**
-     * @see ActionDimension::getVersion()
-     * @return string
-     * @ignore
-     */
-    public function getVersion()
-    {
-        return $this->columnType;
-    }
-
-    /**
-     * Adds a new segment. It automatically sets the SQL segment depending on the column name in case none is set
-     * already.
-     *
-     * @see \Piwik\Columns\Dimension::addSegment()
-     * @param Segment $segment
-     * @api
-     */
-    protected function addSegment(Segment $segment)
-    {
-        $sqlSegment = $segment->getSqlSegment();
-        if (!empty($this->columnName) && empty($sqlSegment)) {
-            $segment->setSqlSegment($this->tableName . '.' . $this->columnName);
-        }
-
-        parent::addSegment($segment);
-    }
+    protected $dbTableName = 'log_conversion';
+    protected $category = 'Goals_Conversion';
 
     /**
      * Get all conversion dimensions that are defined by all activated plugins.
diff --git a/core/Plugin/Dimension/VisitDimension.php b/core/Plugin/Dimension/VisitDimension.php
index c59495d2aa..f6e3f19205 100644
--- a/core/Plugin/Dimension/VisitDimension.php
+++ b/core/Plugin/Dimension/VisitDimension.php
@@ -13,12 +13,11 @@ use Piwik\Cache as PiwikCache;
 use Piwik\Columns\Dimension;
 use Piwik\Common;
 use Piwik\Db;
+use Piwik\DbHelper;
 use Piwik\Plugin\Manager as PluginManager;
-use Piwik\Plugin\Segment;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
 use Piwik\Tracker\Action;
-use Piwik\Tracker;
 use Piwik\Plugin;
 use Exception;
 
@@ -39,38 +38,9 @@ abstract class VisitDimension extends Dimension
 {
     const INSTALLER_PREFIX = 'log_visit.';
 
-    private $tableName = 'log_visit';
+    protected $dbTableName = 'log_visit';
+    protected $category = 'General_Visitors';
 
-    /**
-     * Installs the visit dimension in case it is not installed yet. The installation is already implemented based on
-     * the {@link $columnName} and {@link $columnType}. If you want to perform additional actions beside adding the
-     * column to the database - for instance adding an index - you can overwrite this method. We recommend to call
-     * this parent method to get the minimum required actions and then add further custom actions since this makes sure
-     * the column will be installed correctly. We also recommend to change the default install behavior only if really
-     * needed. FYI: We do not directly execute those alter table statements here as we group them together with several
-     * other alter table statements do execute those changes in one step which results in a faster installation. The
-     * column will be added to the `log_visit` MySQL table.
-     *
-     * Example:
-     * ```
-    public function install()
-    {
-        $changes = parent::install();
-        $changes['log_visit'][] = "ADD INDEX index_idsite_servertime ( idsite, server_time )";
-
-        return $changes;
-    }
-    ```
-     *
-     * @return array An array containing the table name as key and an array of MySQL alter table statements that should
-     *               be executed on the given table. Example:
-     * ```
-    array(
-        'log_visit' => array("ADD COLUMN `$this->columnName` $this->columnType", "ADD INDEX ...")
-    );
-    ```
-     * @api
-     */
     public function install()
     {
         if (empty($this->columnType) || empty($this->columnName)) {
@@ -78,7 +48,7 @@ abstract class VisitDimension extends Dimension
         }
 
         $changes = array(
-            $this->tableName => array("ADD COLUMN `$this->columnName` $this->columnType")
+            $this->dbTableName => array("ADD COLUMN `$this->columnName` $this->columnType")
         );
 
         if ($this->isHandlingLogConversion()) {
@@ -90,19 +60,20 @@ abstract class VisitDimension extends Dimension
 
     /**
      * @see ActionDimension::update()
-     * @param array $conversionColumns An array of currently installed columns in the conversion table.
      * @return array
      * @ignore
      */
-    public function update($conversionColumns)
+    public function update()
     {
         if (!$this->columnType) {
             return array();
         }
 
+        $conversionColumns = DbHelper::getTableColumns(Common::prefixTable('log_conversion'));
+
         $changes = array();
 
-        $changes[$this->tableName] = array("MODIFY COLUMN `$this->columnName` $this->columnType");
+        $changes[$this->dbTableName] = array("MODIFY COLUMN `$this->columnName` $this->columnType");
 
         $handlingConversion  = $this->isHandlingLogConversion();
         $hasConversionColumn = array_key_exists($this->columnName, $conversionColumns);
@@ -119,7 +90,6 @@ abstract class VisitDimension extends Dimension
     }
 
     /**
-     * @see ActionDimension::getVersion()
      * @return string
      * @ignore
      */
@@ -152,7 +122,7 @@ abstract class VisitDimension extends Dimension
         }
 
         try {
-            $sql = "ALTER TABLE `" . Common::prefixTable($this->tableName) . "` DROP COLUMN `$this->columnName`";
+            $sql = "ALTER TABLE `" . Common::prefixTable($this->dbTableName) . "` DROP COLUMN `$this->columnName`";
             Db::exec($sql);
         } catch (Exception $e) {
             if (!Db::get()->isErrNo($e, '1091')) {
@@ -175,23 +145,6 @@ abstract class VisitDimension extends Dimension
     }
 
     /**
-     * Adds a new segment. It automatically sets the SQL segment depending on the column name in case none is set
-     * already.
-     * @see \Piwik\Columns\Dimension::addSegment()
-     * @param Segment $segment
-     * @api
-     */
-    protected function addSegment(Segment $segment)
-    {
-        $sqlSegment = $segment->getSqlSegment();
-        if (!empty($this->columnName) && empty($sqlSegment)) {
-            $segment->setSqlSegment('log_visit.' . $this->columnName);
-        }
-
-        parent::addSegment($segment);
-    }
-
-    /**
      * Sometimes you may want to make sure another dimension is executed before your dimension so you can persist
      * this dimensions' value depending on the value of other dimensions. You can do this by defining an array of
      * dimension names. If you access any value of any other column within your events, you should require them here.