Better segment editor and fixes (#12040)

* column tweak

* fix install

* more tweaks

* rename column to dimension

* various fixes

* added new control expandable select

* starting to refactor segment selector

* make segment editor work again

* use translation keys

* defined some metrics

* set types

* simplify

* simplify

* fix join generator

* add possibility to use custom join table names when using query builder and it uses an inner query

* fix bug in query selector when selecting same field name from different tables twice

* more metadata

* more tweaks

* improve selector

* add possibility to use custom entity names

* also processed archived metrics

* generate sql filter, suggested values callback, and accept values automatically for columns with enums

* several tweaks

* focus search field when opening it

* various tweaks

* added missing method

* format and fix more metadata

* more fixes

* better definition

* define custom filter

* fix definition

* fix various tests

* fix more tests

* fix bug in logquery builder

* fix referrerurl segment was missing

* fix some tests

* fix more tests

* add group

* refactor for better definition

* fix a bug in log query builder when similar columns are used in archiver

* add goal metrics

* various fixes

* make datatable row more flexible

* various fixes and visualization enhancements

* simply segment editor and make it smaller

* remove trailing comma

* various fixes and added new dimension

* fix formatting of returning customer

* added missing primary key

* fixes

* various fixes and improvements

* make sure to update segment definition when selecting a value from auto complete list

* various fixes and more metrics

* more metrics

* more dimensions and fixes

* fix some tests

* fix some integration tests

* update submodule

* fix some system tests

* fix ui tests

* trigger new test run

* fix more ui tests

* fix system tests

* update submodule

* fix categories

* sort segments by category for more consistency

* add custom variables

* some translations and fixes

* add minute segment

* more segments

* added plurals

* added some docs

* fix test

* fix tests

* fix tests

* added suggested values

* fix some tests

* various fixes

* fix more tests

* allow to select segments on any site

* make sure to include file

* added doc block

* fix some system tests

* fix most system tests

* fix ui test

* fix system test

* adjust examples

* added more tests and docs

* no metrics for these dimensions

* added developer changelog and made some classes public api

* some fixes for entity names

* add possibility to set format metrics in test

* more consistency in defining the name

* get idsites only if provided

* fix integration tests

* added another segment for visit start hour and visit start minute

* more clear name for segment

* use old segment name to not break bc

* various fixes

* more test fixes

* fix no suggested values for new segment

* add event value

* for boolean dimensions only sum metric

* update available widgets when updating reporting menu

* Add new segments in developer changelog + typo

* fix system tests

* fix screenshot test

28 files changed, 458 insertions, 133 deletions

diff --git a/plugins/CoreHome/Columns/IdSite.php b/plugins/CoreHome/Columns/IdSite.php
index 996cae629b..f757b5bf1c 100644
--- a/plugins/CoreHome/Columns/IdSite.php
+++ b/plugins/CoreHome/Columns/IdSite.php
@@ -8,10 +8,10 @@
  */
 namespace Piwik\Plugins\CoreHome\Columns;
 
+use Piwik\Columns\Join\SiteNameJoin;
 use Piwik\Plugin\Dimension\VisitDimension;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\Request;
-use Piwik\Tracker;
 use Piwik\Tracker\Visitor;
 
 class IdSite extends VisitDimension
@@ -23,6 +23,13 @@ class IdSite extends VisitDimension
     // INDEX(idsite, config_id, visit_last_action_time) and we maybe not be sure whether config_id already exists at
     // installing point (we do not know whether visit_last_action_time or idsite column would be added first).
 
+    protected $type = self::TYPE_TEXT;
+
+    public function getDbColumnJoin()
+    {
+        return new SiteNameJoin();
+    }
+
     /**
      * @param Request $request
      * @param Visitor $visitor
diff --git a/plugins/CoreHome/Columns/LinkVisitActionId.php b/plugins/CoreHome/Columns/LinkVisitActionId.php
new file mode 100644
index 0000000000..6cc55c04bb
--- /dev/null
+++ b/plugins/CoreHome/Columns/LinkVisitActionId.php
@@ -0,0 +1,42 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\CoreHome\Columns;
+
+use Piwik\Columns\DimensionMetricFactory;
+use Piwik\Columns\MetricsList;
+use Piwik\Piwik;
+use Piwik\Plugin\ArchivedMetric;
+use Piwik\Plugin\Dimension\ActionDimension;
+use Piwik\Plugin\Dimension\VisitDimension;
+use Piwik\Plugin\Segment;
+
+
+class LinkVisitActionId extends ActionDimension
+{
+    protected $columnName = 'idlink_va';
+    protected $acceptValues = 'Any integer.';
+    protected $category = 'General_Actions';
+    protected $nameSingular = 'General_Actions';
+    protected $metricId = 'hits';
+    protected $type = self::TYPE_NUMBER;
+
+    public function configureMetrics(MetricsList $metricsList, DimensionMetricFactory $dimensionMetricFactory)
+    {
+        $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_UNIQUE);
+        $metric->setTranslatedName(Piwik::translate('General_ColumnHits'));
+        $metric->setName('hits');
+        $metricsList->addMetric($metric);
+
+        $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_UNIQUE);
+        $metric->setTranslatedName(Piwik::translate('General_ColumnPageviews'));
+        $metric->setDocumentation(Piwik::translate('General_ColumnPageviewsDocumentation'));
+        $metric->setName('pageviews');
+        $metricsList->addMetric($metric);
+    }
+}
\ No newline at end of file
diff --git a/plugins/CoreHome/Columns/ServerMinute.php b/plugins/CoreHome/Columns/ServerMinute.php
new file mode 100644
index 0000000000..ee095dcb3c
--- /dev/null
+++ b/plugins/CoreHome/Columns/ServerMinute.php
@@ -0,0 +1,35 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\CoreHome\Columns;
+
+use Piwik\Metrics\Formatter;
+use Piwik\Plugin\Dimension\ActionDimension;
+
+class ServerMinute extends ActionDimension
+{
+    protected $columnName = 'server_time';
+    protected $segmentName = 'actionServerMinute';
+    protected $sqlSegment = 'MINUTE(log_link_visit_action.server_time)';
+    protected $nameSingular = 'VisitTime_ColumnServerMinute';
+    protected $type = self::TYPE_DATETIME;
+    protected $acceptValues = '0, 1, 2, 3, ..., 56, 57, 58, 59';
+
+    public function __construct()
+    {
+        $this->suggestedValuesCallback = function ($idSite, $maxValuesToReturn) {
+            return range(0, min(59, $maxValuesToReturn));
+        };
+    }
+
+    public function formatValue($value, $idSite, Formatter $formatter)
+    {
+        return $value;
+    }
+
+}
diff --git a/plugins/CoreHome/Columns/ServerTime.php b/plugins/CoreHome/Columns/ServerTime.php
index 297c410058..4e308786f1 100644
--- a/plugins/CoreHome/Columns/ServerTime.php
+++ b/plugins/CoreHome/Columns/ServerTime.php
@@ -9,17 +9,34 @@
 namespace Piwik\Plugins\CoreHome\Columns;
 
 use Piwik\Date;
-use Piwik\Db;
+use Piwik\Metrics\Formatter;
 use Piwik\Plugin\Dimension\ActionDimension;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
-use Piwik\Tracker;
+
+require_once PIWIK_INCLUDE_PATH . '/plugins/VisitTime/functions.php';
 
 class ServerTime extends ActionDimension
 {
     protected $columnName = 'server_time';
     protected $columnType = 'DATETIME NOT NULL';
+    protected $segmentName = 'actionServerHour';
+    protected $sqlSegment = 'HOUR(log_link_visit_action.server_time)';
+    protected $nameSingular = 'VisitTime_ColumnServerHour';
+    protected $type = self::TYPE_DATETIME;
+
+    public function __construct()
+    {
+        $this->suggestedValuesCallback = function ($idSite, $maxValuesToReturn) {
+            return range(0, min(23, $maxValuesToReturn));
+        };
+    }
+
+    public function formatValue($value, $idSite, Formatter $formatter)
+    {
+        return \Piwik\Plugins\VisitTime\getTimeLabel($value);
+    }
 
     public function install()
     {
diff --git a/plugins/CoreHome/Columns/UserId.php b/plugins/CoreHome/Columns/UserId.php
index 50d1be09bc..c5423ae93a 100644
--- a/plugins/CoreHome/Columns/UserId.php
+++ b/plugins/CoreHome/Columns/UserId.php
@@ -12,9 +12,7 @@ use Piwik\Cache;
 use Piwik\DataTable;
 use Piwik\DataTable\Map;
 use Piwik\Metrics;
-use Piwik\Piwik;
 use Piwik\Plugin\Dimension\VisitDimension;
-use Piwik\Plugin\Segment;
 use Piwik\Plugins\VisitsSummary\API as VisitsSummaryApi;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
@@ -29,25 +27,18 @@ class UserId extends VisitDimension
      * @var string
      */
     protected $columnName = 'user_id';
+    protected $type = self::TYPE_TEXT;
+    protected $allowAnonymous = false;
+    protected $segmentName = 'userId';
+    protected $nameSingular = 'General_UserId';
+    protected $namePlural = 'General_UserIds';
+    protected $acceptValues = 'any non empty unique string identifying the user (such as an email address or a username).';
 
     /**
      * @var string
      */
     protected $columnType = 'VARCHAR(200) NULL';
 
-    protected function configureSegments()
-    {
-        $segment = new Segment();
-        $segment->setType('dimension');
-        $segment->setSegment('userId');
-        $segment->setCategory(Piwik::translate('General_Visit'));
-        $segment->setName('General_UserId');
-        $segment->setAcceptedValues('any non empty unique string identifying the user (such as an email address or a username).');
-        $segment->setSqlSegment('log_visit.user_id');
-        $segment->setRequiresAtLeastViewAccess(true);
-        $this->addSegment($segment);
-    }
-
     /**
      * @param Request $request
      * @param Visitor $visitor
diff --git a/plugins/CoreHome/Columns/VisitFirstActionMinute.php b/plugins/CoreHome/Columns/VisitFirstActionMinute.php
new file mode 100644
index 0000000000..c3e16a9215
--- /dev/null
+++ b/plugins/CoreHome/Columns/VisitFirstActionMinute.php
@@ -0,0 +1,45 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\CoreHome\Columns;
+
+use Piwik\Columns\DimensionMetricFactory;
+use Piwik\Columns\MetricsList;
+use Piwik\Plugin\Dimension\VisitDimension;
+use Piwik\Metrics\Formatter;
+
+require_once PIWIK_INCLUDE_PATH . '/plugins/VisitTime/functions.php';
+
+class VisitFirstActionMinute extends VisitDimension
+{
+    protected $columnName = 'visit_first_action_time';
+    protected $type = self::TYPE_DATETIME;
+
+    protected $sqlSegment = 'HOUR(log_visit.visit_first_action_time)';
+    protected $segmentName = 'visitStartServerMinute';
+    protected $acceptValues = '0, 1, 2, 3, ..., 56, 57, 58, 59';
+    protected $nameSingular = 'VisitTime_ColumnVisitStartServerMinute';
+
+    public function __construct()
+    {
+        $this->suggestedValuesCallback = function ($idSite, $maxValuesToReturn) {
+            return range(0, min(59, $maxValuesToReturn));
+        };
+    }
+
+    public function configureMetrics(MetricsList $metricsList, DimensionMetricFactory $dimensionMetricFactory)
+    {
+        // no metrics to be generated
+    }
+
+    public function formatValue($value, $idSite, Formatter $formatter)
+    {
+        return $value;
+    }
+
+}
\ No newline at end of file
diff --git a/plugins/CoreHome/Columns/VisitFirstActionTime.php b/plugins/CoreHome/Columns/VisitFirstActionTime.php
index 3985d53339..f9318d6771 100644
--- a/plugins/CoreHome/Columns/VisitFirstActionTime.php
+++ b/plugins/CoreHome/Columns/VisitFirstActionTime.php
@@ -12,13 +12,33 @@ use Piwik\Date;
 use Piwik\Plugin\Dimension\VisitDimension;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\Request;
-use Piwik\Tracker;
 use Piwik\Tracker\Visitor;
+use Piwik\Metrics\Formatter;
+
+require_once PIWIK_INCLUDE_PATH . '/plugins/VisitTime/functions.php';
 
 class VisitFirstActionTime extends VisitDimension
 {
     protected $columnName = 'visit_first_action_time';
     protected $columnType = 'DATETIME NOT NULL';
+    protected $type = self::TYPE_DATETIME;
+
+    protected $sqlSegment = 'HOUR(log_visit.visit_first_action_time)';
+    protected $segmentName = 'visitStartServerHour';
+    protected $acceptValues = '0, 1, 2, 3, ..., 20, 21, 22, 23';
+    protected $nameSingular = 'VisitTime_ColumnVisitStartServerHour';
+
+    public function __construct()
+    {
+        $this->suggestedValuesCallback = function ($idSite, $maxValuesToReturn) {
+            return range(0, min(23, $maxValuesToReturn));
+        };
+    }
+
+    public function formatValue($value, $idSite, Formatter $formatter)
+    {
+        return $value;
+    }
 
     /**
      * @param Request $request
diff --git a/plugins/CoreHome/Columns/VisitGoalBuyer.php b/plugins/CoreHome/Columns/VisitGoalBuyer.php
index 779d5fd5be..5c6f908173 100644
--- a/plugins/CoreHome/Columns/VisitGoalBuyer.php
+++ b/plugins/CoreHome/Columns/VisitGoalBuyer.php
@@ -8,6 +8,7 @@
  */
 namespace Piwik\Plugins\CoreHome\Columns;
 
+use Piwik\Metrics\Formatter;
 use Piwik\Piwik;
 use Piwik\Plugin\Dimension\VisitDimension;
 use Piwik\Plugins\CoreHome\Segment;
@@ -33,19 +34,39 @@ class VisitGoalBuyer extends VisitDimension
 
     protected $columnName = 'visit_goal_buyer';
     protected $columnType = 'TINYINT(1) NULL';
+    protected $segmentName = 'visitEcommerceStatus';
+    protected $nameSingular = 'General_EcommerceVisitStatusDesc';
+    protected $type = self::TYPE_ENUM;
 
-    protected function configureSegments()
+    public function __construct()
     {
         $example = Piwik::translate('General_EcommerceVisitStatusEg', '"&segment=visitEcommerceStatus==ordered,visitEcommerceStatus==orderedThenAbandonedCart"');
-        $acceptedValues = implode(", ", self::$visitEcommerceStatus) . '. ' . $example;
+        $this->acceptValues = implode(", ", self::$visitEcommerceStatus) . '. ' . $example;
+    }
+
+    public function formatValue($value, $idSite, Formatter $formatter)
+    {
+        switch ($value) {
+            case 'ordered':
+            case self::TYPE_BUYER_ORDERED:
+                return Piwik::translate('CoreHome_VisitStatusOrdered');
+            case 'abandonedCart':
+            case self::TYPE_BUYER_OPEN_CART:
+                return Piwik::translate('Goals_AbandonedCart');
+            case 'orderedThenAbandonedCart':
+            case self::TYPE_BUYER_ORDERED_AND_OPEN_CART:
+                return Piwik::translate('CoreHome_VisitStatusOrderedThenAbandoned');
+            case 'none';
+            case self::TYPE_BUYER_NONE:
+                return Piwik::translate('UserCountryMap_None');
+        }
 
-        $segment = new Segment();
-        $segment->setSegment('visitEcommerceStatus');
-        $segment->setName('General_EcommerceVisitStatusDesc');
-        $segment->setAcceptedValues($acceptedValues);
-        $segment->setSqlFilterValue(__NAMESPACE__ . '\VisitGoalBuyer::getVisitEcommerceStatus');
+        return $value;
+    }
 
-        $this->addSegment($segment);
+    public function getEnumColumnValues()
+    {
+        return self::$visitEcommerceStatus;
     }
 
     /**
@@ -83,17 +104,6 @@ class VisitGoalBuyer extends VisitDimension
         return false;
     }
 
-    public static function getVisitEcommerceStatus($status)
-    {
-        $id = array_search($status, self::$visitEcommerceStatus);
-
-        if ($id === false) {
-            throw new \Exception("Invalid 'visitEcommerceStatus' segment value $status");
-        }
-
-        return $id;
-    }
-
     /**
      * @ignore
      */
diff --git a/plugins/CoreHome/Columns/VisitGoalConverted.php b/plugins/CoreHome/Columns/VisitGoalConverted.php
index 1c0cc40e65..26d87b7835 100644
--- a/plugins/CoreHome/Columns/VisitGoalConverted.php
+++ b/plugins/CoreHome/Columns/VisitGoalConverted.php
@@ -8,8 +8,11 @@
  */
 namespace Piwik\Plugins\CoreHome\Columns;
 
+use Piwik\Columns\DimensionMetricFactory;
+use Piwik\Columns\MetricsList;
+use Piwik\Piwik;
+use Piwik\Plugin\ComputedMetric;
 use Piwik\Plugin\Dimension\VisitDimension;
-use Piwik\Plugins\CoreHome\Segment;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
@@ -18,14 +21,21 @@ class VisitGoalConverted extends VisitDimension
 {
     protected $columnName = 'visit_goal_converted';
     protected $columnType = 'TINYINT(1) NULL';
+    protected $type = self::TYPE_BOOL;
+    protected $segmentName = 'visitConverted';
+    protected $nameSingular = 'General_VisitConvertedGoal';
+    protected $acceptValues = '0, 1';
 
-    protected function configureSegments()
+    public function configureMetrics(MetricsList $metricsList, DimensionMetricFactory $dimensionMetricFactory)
     {
-        $segment = new Segment();
-        $segment->setSegment('visitConverted');
-        $segment->setName('General_VisitConvertedGoal');
-        $segment->setAcceptedValues('0, 1');
-        $this->addSegment($segment);
+        $metric1 = $dimensionMetricFactory->createCustomMetric('nb_visits_converted', Piwik::translate('General_ColumnVisitsWithConversions'), 'sum(case %s when 1 then 1 else 0 end)');
+        $metricsList->addMetric($metric1);
+
+        $metric = $dimensionMetricFactory->createComputedMetric($metric1->getName(), 'nb_visits', ComputedMetric::AGGREGATION_RATE);
+        $metric->setTranslatedName(Piwik::translate('General_ColumnConversionRate'));
+        $metric->setDocumentation(Piwik::translate('General_ColumnConversionRateDocumentation'));
+        $metric->setName('visits_conversion_rate');
+        $metricsList->addMetric($metric);
     }
 
     /**
diff --git a/plugins/CoreHome/Columns/VisitId.php b/plugins/CoreHome/Columns/VisitId.php
index 9add50888a..b87f1b58d4 100644
--- a/plugins/CoreHome/Columns/VisitId.php
+++ b/plugins/CoreHome/Columns/VisitId.php
@@ -8,7 +8,10 @@
 
 namespace Piwik\Plugins\CoreHome\Columns;
 
+use Piwik\Columns\DimensionMetricFactory;
+use Piwik\Columns\MetricsList;
 use Piwik\Piwik;
+use Piwik\Plugin\ArchivedMetric;
 use Piwik\Plugin\Dimension\VisitDimension;
 use Piwik\Plugin\Segment;
 
@@ -18,18 +21,28 @@ use Piwik\Plugin\Segment;
  */
 class VisitId extends VisitDimension
 {
+    protected $columnName = 'idvisit';
+    protected $acceptValues = 'Any integer.';
+    protected $nameSingular = 'General_Visit';
+    protected $namePlural = 'General_ColumnNbVisits';
+    protected $segmentName = 'visitId';
+    protected $allowAnonymous = false;
+    protected $metricId = 'visits';
+    protected $type = self::TYPE_TEXT;
+
     protected function configureSegments()
     {
-        parent::configureSegments();
-
         $segment = new Segment();
-        $segment->setType('dimension');
-        $segment->setCategory(Piwik::translate('General_Visit'));
-        $segment->setName(Piwik::translate('General_Visit') . " ID");
-        $segment->setSegment('visitId');
-        $segment->setAcceptedValues('Any integer.');
-        $segment->setSqlSegment('log_visit.idvisit');
-        $segment->setRequiresAtLeastViewAccess(true);
+        $segment->setName('General_VisitId');
         $this->addSegment($segment);
     }
+
+    public function configureMetrics(MetricsList $metricsList, DimensionMetricFactory $dimensionMetricFactory)
+    {
+        $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_UNIQUE);
+        $metric->setTranslatedName(Piwik::translate('General_ColumnNbVisits'));
+        $metric->setDocumentation(Piwik::translate('General_ColumnNbVisitsDocumentation'));
+        $metric->setName('nb_visits');
+        $metricsList->addMetric($metric);
+    }
 }
\ No newline at end of file
diff --git a/plugins/CoreHome/Columns/VisitIp.php b/plugins/CoreHome/Columns/VisitIp.php
index 6dd2e5d6b9..ed8b196fd2 100644
--- a/plugins/CoreHome/Columns/VisitIp.php
+++ b/plugins/CoreHome/Columns/VisitIp.php
@@ -8,7 +8,9 @@
 
 namespace Piwik\Plugins\CoreHome\Columns;
 
-use Piwik\Piwik;
+use Piwik\Common;
+use Piwik\Metrics\Formatter;
+use Piwik\Network\IPUtils;
 use Piwik\Plugin\Dimension\VisitDimension;
 use Piwik\Plugin\Segment;
 
@@ -18,19 +20,26 @@ use Piwik\Plugin\Segment;
  */
 class VisitIp extends VisitDimension
 {
-    protected function configureSegments()
+    protected $columnName = 'location_ip';
+    protected $type = self::TYPE_BINARY;
+    protected $allowAnonymous = false;
+    protected $segmentName = 'visitIp';
+    protected $nameSingular = 'General_VisitorIP';
+    protected $namePlural = 'General_VisitorIPs';
+    protected $acceptValues = '13.54.122.1. </code>Select IP ranges with notation: <code>visitIp>13.54.122.0;visitIp<13.54.122.255';
+    protected $sqlFilterValue = array('Piwik\Network\IPUtils', 'stringToBinaryIP');
+
+    public function formatValue($value, $idSite, Formatter $formatter)
     {
-        parent::configureSegments();
+        $value = Common::hex2bin($value);
+        $value = IPUtils::binaryToStringIP($value);
+        return $value;
+    }
 
+    protected function configureSegments()
+    {
         $segment = new Segment();
-        $segment->setType('metric');
-        $segment->setCategory(Piwik::translate('General_Visit'));
-        $segment->setName('General_VisitorIP');
-        $segment->setSegment('visitIp');
-        $segment->setAcceptedValues('13.54.122.1. </code>Select IP ranges with notation: <code>visitIp>13.54.122.0;visitIp<13.54.122.255');
-        $segment->setSqlSegment('log_visit.location_ip');
-        $segment->setSqlFilterValue(array('Piwik\Network\IPUtils', 'stringToBinaryIP'));
-        $segment->setRequiresAtLeastViewAccess(true);
+        $segment->setType(Segment::TYPE_METRIC); // we cannot remove this for now as it would assign dimension based on text type
         $this->addSegment($segment);
     }
 }
diff --git a/plugins/CoreHome/Columns/VisitLastActionMinute.php b/plugins/CoreHome/Columns/VisitLastActionMinute.php
new file mode 100644
index 0000000000..06611fb2fc
--- /dev/null
+++ b/plugins/CoreHome/Columns/VisitLastActionMinute.php
@@ -0,0 +1,51 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\CoreHome\Columns;
+
+use Piwik\Columns\DimensionMetricFactory;
+use Piwik\Columns\MetricsList;
+use Piwik\Plugin\Dimension\VisitDimension;
+use Piwik\Metrics\Formatter;
+
+require_once PIWIK_INCLUDE_PATH . '/plugins/VisitTime/functions.php';
+
+/**
+ * This dimension holds the best guess for a visit's end time. It is set the last action
+ * time for each visit. `ping=1` requests can be sent to update the dimension value so
+ * it can be a more accurate guess of the time the visitor spent on the site.
+ *
+ * Note: though it is named 'visit last action time' it actually refers to the visit's last action's
+ * end time.
+ */
+class VisitLastActionMinute extends VisitDimension
+{
+    protected $columnName = 'visit_last_action_time';
+    protected $type = self::TYPE_DATETIME;
+    protected $segmentName = 'visitEndServerMinute';
+    protected $nameSingular = 'VisitTime_ColumnVisitEndServerMinute';
+    protected $sqlSegment = 'MINUTE(log_visit.visit_last_action_time)';
+    protected $acceptValues = '0, 1, 2, 3, ..., 56, 57, 58, 59';
+
+    public function __construct()
+    {
+        $this->suggestedValuesCallback = function ($idSite, $maxValuesToReturn) {
+            return range(0, min(59, $maxValuesToReturn));
+        };
+    }
+
+    public function configureMetrics(MetricsList $metricsList, DimensionMetricFactory $dimensionMetricFactory)
+    {
+        // no metrics for this dimension
+    }
+
+    public function formatValue($value, $idSite, Formatter $formatter)
+    {
+        return $value;
+    }
+}
\ No newline at end of file
diff --git a/plugins/CoreHome/Columns/VisitLastActionTime.php b/plugins/CoreHome/Columns/VisitLastActionTime.php
index 4ba0761c2a..3b297fb68b 100644
--- a/plugins/CoreHome/Columns/VisitLastActionTime.php
+++ b/plugins/CoreHome/Columns/VisitLastActionTime.php
@@ -12,8 +12,10 @@ use Piwik\Date;
 use Piwik\Plugin\Dimension\VisitDimension;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\Request;
-use Piwik\Tracker;
 use Piwik\Tracker\Visitor;
+use Piwik\Metrics\Formatter;
+
+require_once PIWIK_INCLUDE_PATH . '/plugins/VisitTime/functions.php';
 
 /**
  * This dimension holds the best guess for a visit's end time. It is set the last action
@@ -26,6 +28,17 @@ use Piwik\Tracker\Visitor;
 class VisitLastActionTime extends VisitDimension
 {
     protected $columnName = 'visit_last_action_time';
+    protected $type = self::TYPE_DATETIME;
+    protected $nameSingular = 'VisitTime_ColumnVisitEndServerHour';
+    protected $sqlSegment = 'HOUR(log_visit.visit_last_action_time)';
+    protected $segmentName = 'visitServerHour';
+    protected $acceptValues = '0, 1, 2, 3, ..., 20, 21, 22, 23';
+
+    public function formatValue($value, $idSite, Formatter $formatter)
+    {
+        return \Piwik\Plugins\VisitTime\getTimeLabel($value);
+    }
+
     // we do not install or define column definition here as we need to create this column when installing as there is
     // an index on it. Currently we do not define the index here... although we could overwrite the install() method
     // and add column 'visit_last_action_time' and add index. Problem is there is also an index
diff --git a/plugins/CoreHome/Columns/VisitTotalTime.php b/plugins/CoreHome/Columns/VisitTotalTime.php
index 87025a90f9..44bdc39958 100644
--- a/plugins/CoreHome/Columns/VisitTotalTime.php
+++ b/plugins/CoreHome/Columns/VisitTotalTime.php
@@ -19,15 +19,9 @@ class VisitTotalTime extends VisitDimension
 {
     protected $columnName = 'visit_total_time';
     protected $columnType = 'INT(11) UNSIGNED NOT NULL';
-
-    protected function configureSegments()
-    {
-        $segment = new Segment();
-        $segment->setSegment('visitDuration');
-        $segment->setName('General_ColumnVisitDuration');
-        $segment->setType(Segment::TYPE_METRIC);
-        $this->addSegment($segment);
-    }
+    protected $segmentName = 'visitDuration';
+    protected $nameSingular = 'General_ColumnVisitDuration';
+    protected $type = self::TYPE_DURATION_S;
 
     /**
      * @param Request $request
diff --git a/plugins/CoreHome/Columns/VisitorDaysSinceFirst.php b/plugins/CoreHome/Columns/VisitorDaysSinceFirst.php
index 3445f21f59..e0ce8fa5be 100644
--- a/plugins/CoreHome/Columns/VisitorDaysSinceFirst.php
+++ b/plugins/CoreHome/Columns/VisitorDaysSinceFirst.php
@@ -9,7 +9,6 @@
 namespace Piwik\Plugins\CoreHome\Columns;
 
 use Piwik\Plugin\Dimension\VisitDimension;
-use Piwik\Plugins\CoreHome\Segment;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
@@ -18,15 +17,9 @@ class VisitorDaysSinceFirst extends VisitDimension
 {
     protected $columnName = 'visitor_days_since_first';
     protected $columnType = 'SMALLINT(5) UNSIGNED NULL';
-
-    protected function configureSegments()
-    {
-        $segment = new Segment();
-        $segment->setType(Segment::TYPE_METRIC);
-        $segment->setSegment('daysSinceFirstVisit');
-        $segment->setName('General_DaysSinceFirstVisit');
-        $this->addSegment($segment);
-    }
+    protected $segmentName = 'daysSinceFirstVisit';
+    protected $nameSingular = 'General_DaysSinceFirstVisit';
+    protected $type = self::TYPE_NUMBER;
 
     /**
      * @param Request $request
diff --git a/plugins/CoreHome/Columns/VisitorDaysSinceOrder.php b/plugins/CoreHome/Columns/VisitorDaysSinceOrder.php
index c126b5957c..ad6334e838 100644
--- a/plugins/CoreHome/Columns/VisitorDaysSinceOrder.php
+++ b/plugins/CoreHome/Columns/VisitorDaysSinceOrder.php
@@ -9,7 +9,6 @@
 namespace Piwik\Plugins\CoreHome\Columns;
 
 use Piwik\Plugin\Dimension\VisitDimension;
-use Piwik\Plugins\CoreHome\Segment;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
@@ -18,16 +17,10 @@ class VisitorDaysSinceOrder extends VisitDimension
 {
     protected $columnName = 'visitor_days_since_order';
     protected $columnType = 'SMALLINT(5) UNSIGNED NULL';
-
-    protected function configureSegments()
-    {
-        $segment = new Segment();
-        $segment->setSegment('daysSinceLastEcommerceOrder');
-        $segment->setName('General_DaysSinceLastEcommerceOrder');
-        $segment->setType(Segment::TYPE_METRIC);
-
-        $this->addSegment($segment);
-    }
+    protected $segmentName = 'daysSinceLastEcommerceOrder';
+    protected $nameSingular = 'General_DaysSinceLastEcommerceOrder';
+    protected $category = 'General_Visitors'; // todo put into ecommerce category?
+    protected $type = self::TYPE_NUMBER;
 
     /**
      * @param Request $request
diff --git a/plugins/CoreHome/Columns/VisitorId.php b/plugins/CoreHome/Columns/VisitorId.php
index ce797ae57a..728e1e3d94 100644
--- a/plugins/CoreHome/Columns/VisitorId.php
+++ b/plugins/CoreHome/Columns/VisitorId.php
@@ -8,9 +8,12 @@
 
 namespace Piwik\Plugins\CoreHome\Columns;
 
+use Piwik\Columns\DimensionMetricFactory;
+use Piwik\Columns\MetricsList;
+use Piwik\Metrics\Formatter;
 use Piwik\Piwik;
+use Piwik\Plugin\ArchivedMetric;
 use Piwik\Plugin\Dimension\VisitDimension;
-use Piwik\Plugin\Segment;
 
 /**
  * Dimension for the log_visit.idvisitor column. This column is added in the CREATE TABLE
@@ -18,19 +21,21 @@ use Piwik\Plugin\Segment;
  */
 class VisitorId extends VisitDimension
 {
-    protected function configureSegments()
-    {
-        parent::configureSegments();
+    protected $columnName = 'idvisitor';
+    protected $metricId = 'visitors';
+    protected $nameSingular = 'General_VisitorID';
+    protected $namePlural = 'General_Visitors';
+    protected $segmentName = 'visitorId';
+    protected $acceptValues = '34c31e04394bdc63 - any 16 Hexadecimal chars ID, which can be fetched using the Tracking API function getVisitorId()';
+    protected $allowAnonymous = false;
+    protected $sqlFilterValue = array('Piwik\Common', 'convertVisitorIdToBin');
+    protected $type = self::TYPE_BINARY;
 
-        $segment = new Segment();
-        $segment->setType('dimension');
-        $segment->setCategory(Piwik::translate('General_Visit'));
-        $segment->setName('General_VisitorID');
-        $segment->setSegment('visitorId');
-        $segment->setAcceptedValues('34c31e04394bdc63 - any 16 Hexadecimal chars ID, which can be fetched using the Tracking API function getVisitorId()');
-        $segment->setSqlSegment('log_visit.idvisitor');
-        $segment->setSqlFilterValue(array('Piwik\Common', 'convertVisitorIdToBin'));
-        $segment->setRequiresAtLeastViewAccess(true);
-        $this->addSegment($segment);
+    public function configureMetrics(MetricsList $metricsList, DimensionMetricFactory $dimensionMetricFactory)
+    {
+        $metric = $dimensionMetricFactory->createMetric(ArchivedMetric::AGGREGATION_UNIQUE);
+        $metric->setTranslatedName(Piwik::translate('General_ColumnNbUniqVisitors'));
+        $metric->setName('nb_uniq_visitors');
+        $metricsList->addMetric($metric);
     }
 }
diff --git a/plugins/CoreHome/Columns/VisitorReturning.php b/plugins/CoreHome/Columns/VisitorReturning.php
index ca1ef11652..862fb12c6e 100644
--- a/plugins/CoreHome/Columns/VisitorReturning.php
+++ b/plugins/CoreHome/Columns/VisitorReturning.php
@@ -8,9 +8,9 @@
  */
 namespace Piwik\Plugins\CoreHome\Columns;
 
+use Piwik\Metrics\Formatter;
 use Piwik\Piwik;
 use Piwik\Plugin\Dimension\VisitDimension;
-use Piwik\Plugins\CoreHome\Segment;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
@@ -23,22 +23,44 @@ class VisitorReturning extends VisitDimension
 
     protected $columnName = 'visitor_returning';
     protected $columnType = 'TINYINT(1) NULL';
+    protected $segmentName = 'visitorType';
+    protected $nameSingular = 'General_VisitType';
+    protected $namePlural = 'General_VisitTypes';
     protected $conversionField = true;
+    protected $type = self::TYPE_ENUM;
 
-    protected function configureSegments()
+    public function __construct()
     {
-        $acceptedValues  = 'new, returning, returningCustomer. ';
-        $acceptedValues .= Piwik::translate('General_VisitTypeExample', '"&segment=visitorType==returning,visitorType==returningCustomer"');
-
-        $segment = new Segment();
-        $segment->setSegment('visitorType');
-        $segment->setName('General_VisitType');
-        $segment->setAcceptedValues($acceptedValues);
-        $segment->setSqlFilterValue(function ($type) {
+        $this->acceptValues  = 'new, returning, returningCustomer. ';
+        $this->acceptValues .= Piwik::translate('General_VisitTypeExample', '"&segment=visitorType==returning,visitorType==returningCustomer"');
+        $this->sqlFilterValue = function ($type) {
+            if (is_numeric($type)) {
+                return $type;
+            }
             return $type == "new" ? 0 : ($type == "returning" ? 1 : 2);
-        });
+        };
+    }
+
+    public function formatValue($value, $idSite, Formatter $formatter)
+    {
+        if ($value === 1 || $value === '1' || $value === 'returning') {
+            return Piwik::translate('CoreHome_VisitTypeReturning');
+        } elseif ($value === 2 || $value === '2' || $value === 'returningCustomer'){
+            return Piwik::translate('CoreHome_VisitTypeReturningCustomer');
+        } elseif ($value === 0 || $value === '0' || $value === 'new'){
+            return Piwik::translate('General_New');
+        }
 
-        $this->addSegment($segment);
+        return $value;
+    }
+
+    public function getEnumColumnValues()
+    {
+        return array(
+            self::IS_RETURNING_CUSTOMER => 'returningCustomer',
+            self::IS_RETURNING => 'returning',
+            self::IS_NEW => 'new',
+        );
     }
 
     /**
@@ -49,7 +71,6 @@ class VisitorReturning extends VisitDimension
      */
     public function onNewVisit(Request $request, Visitor $visitor, $action)
     {
-
         $daysSinceLastOrder = $request->getDaysSinceLastOrder();
         $isReturningCustomer = ($daysSinceLastOrder !== false);
 
diff --git a/plugins/CoreHome/Columns/VisitsCount.php b/plugins/CoreHome/Columns/VisitsCount.php
index 584b242ae4..c334f175bb 100644
--- a/plugins/CoreHome/Columns/VisitsCount.php
+++ b/plugins/CoreHome/Columns/VisitsCount.php
@@ -8,8 +8,9 @@
  */
 namespace Piwik\Plugins\CoreHome\Columns;
 
+use Piwik\Columns\DimensionMetricFactory;
+use Piwik\Columns\MetricsList;
 use Piwik\Plugin\Dimension\VisitDimension;
-use Piwik\Plugins\CoreHome\Segment;
 use Piwik\Tracker\Action;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
@@ -18,14 +19,13 @@ class VisitsCount extends VisitDimension
 {
     protected $columnName = 'visitor_count_visits';
     protected $columnType = 'INT(11) UNSIGNED NOT NULL';
+    protected $segmentName = 'visitCount';
+    protected $nameSingular = 'General_NumberOfVisits';
+    protected $type = self::TYPE_NUMBER;
 
-    protected function configureSegments()
+    public function configureMetrics(MetricsList $metricsList, DimensionMetricFactory $dimensionMetricFactory)
     {
-        $segment = new Segment();
-        $segment->setType(Segment::TYPE_METRIC);
-        $segment->setSegment('visitCount');
-        $segment->setName('General_NumberOfVisits');
-        $this->addSegment($segment);
+        // no metrics for this dimension, it would be rather confusing I think
     }
 
     /**
diff --git a/plugins/CoreHome/CoreHome.php b/plugins/CoreHome/CoreHome.php
index be56c144f8..ff800ebe1e 100644
--- a/plugins/CoreHome/CoreHome.php
+++ b/plugins/CoreHome/CoreHome.php
@@ -7,6 +7,10 @@
  *
  */
 namespace Piwik\Plugins\CoreHome;
+use Piwik\Columns\ComputedMetricFactory;
+use Piwik\Columns\MetricsList;
+use Piwik\Plugin\ArchivedMetric;
+use Piwik\Plugin\ComputedMetric;
 
 /**
  *
@@ -21,7 +25,7 @@ class CoreHome extends \Piwik\Plugin
     const WIDGET_CONTAINER_LAYOUT_BY_DIMENSION = 'ByDimension';
 
     /**
-     * @see Piwik\Plugin::registerEvents
+     * @see \Piwik\Plugin::registerEvents
      */
     public function registerEvents()
     {
@@ -30,9 +34,27 @@ class CoreHome extends \Piwik\Plugin
             'AssetManager.getJavaScriptFiles'        => 'getJsFiles',
             'AssetManager.filterMergedJavaScripts'   => 'filterMergedJavaScripts',
             'Translate.getClientSideTranslationKeys' => 'getClientSideTranslationKeys',
+            'Metric.addComputedMetrics'              => 'addComputedMetrics'
         );
     }
 
+    public function addComputedMetrics(MetricsList $list, ComputedMetricFactory $computedMetricFactory)
+    {
+        $metrics = $list->getMetrics();
+        foreach ($metrics as $metric) {
+            if ($metric instanceof ArchivedMetric && $metric->getDimension()) {
+                $metricName = $metric->getName();
+                if ($metric->getDbTableName() === 'log_visit'
+                    && $metricName !== 'nb_uniq_visitors'
+                    && $metricName !== 'nb_visits'
+                    && strpos($metricName, ArchivedMetric::AGGREGATION_SUM_PREFIX) === 0) {
+                    $metric = $computedMetricFactory->createComputedMetric($metric->getName(), 'nb_visits', ComputedMetric::AGGREGATION_AVG);
+                    $list->addMetric($metric);
+                }
+            }
+        }
+    }
+
     public function filterMergedJavaScripts(&$mergedContent)
     {
         $mergedContent = preg_replace('/(sourceMappingURL=(.*?).map)/', '', $mergedContent);
diff --git a/plugins/CoreHome/Tracker/LogTable/Conversion.php b/plugins/CoreHome/Tracker/LogTable/Conversion.php
index b579b32d81..2a75d52c41 100644
--- a/plugins/CoreHome/Tracker/LogTable/Conversion.php
+++ b/plugins/CoreHome/Tracker/LogTable/Conversion.php
@@ -26,4 +26,9 @@ class Conversion extends LogTable
     {
         return 'idvisit';
     }
+
+    public function getPrimaryKey()
+    {
+        return array('idvisit', 'idgoal', 'buster');
+    }
 }
\ No newline at end of file
diff --git a/plugins/CoreHome/Tracker/LogTable/ConversionItem.php b/plugins/CoreHome/Tracker/LogTable/ConversionItem.php
index cf24614458..8423ff8010 100644
--- a/plugins/CoreHome/Tracker/LogTable/ConversionItem.php
+++ b/plugins/CoreHome/Tracker/LogTable/ConversionItem.php
@@ -26,5 +26,10 @@ class ConversionItem extends LogTable
     {
         return 'idvisit';
     }
-    
+
+    public function getPrimaryKey()
+    {
+        return array('idvisit', 'idorder', 'idaction_sku');
+    }
+
 }
diff --git a/plugins/CoreHome/Tracker/LogTable/LinkVisitAction.php b/plugins/CoreHome/Tracker/LogTable/LinkVisitAction.php
index 8dabf66a66..c2090a1c8c 100644
--- a/plugins/CoreHome/Tracker/LogTable/LinkVisitAction.php
+++ b/plugins/CoreHome/Tracker/LogTable/LinkVisitAction.php
@@ -31,4 +31,9 @@ class LinkVisitAction extends LogTable
     {
         return 'idvisit';
     }
+
+    public function getPrimaryKey()
+    {
+        return array('idlink_va');
+    }
 }
diff --git a/plugins/CoreHome/Tracker/LogTable/Visit.php b/plugins/CoreHome/Tracker/LogTable/Visit.php
index a72fb24d11..62877f1cd1 100644
--- a/plugins/CoreHome/Tracker/LogTable/Visit.php
+++ b/plugins/CoreHome/Tracker/LogTable/Visit.php
@@ -32,4 +32,8 @@ class Visit extends LogTable
         return true;
     }
 
+    public function getPrimaryKey()
+    {
+        return array('idvisit');
+    }
 }
\ No newline at end of file
diff --git a/plugins/CoreHome/angularjs/common/services/piwik-api.js b/plugins/CoreHome/angularjs/common/services/piwik-api.js
index 67f9756f4f..65127ece8a 100644
--- a/plugins/CoreHome/angularjs/common/services/piwik-api.js
+++ b/plugins/CoreHome/angularjs/common/services/piwik-api.js
@@ -214,7 +214,7 @@ var hasBlockedContent = false;
             }
 
             for (var key in defaultParams) {
-                if (!getParamsToMixin[key] && !postParams[key] && defaultParams[key]) {
+                if (!(key in getParamsToMixin) && !(key in postParams) && defaultParams[key]) {
                     getParamsToMixin[key] = defaultParams[key];
                 }
             }
diff --git a/plugins/CoreHome/angularjs/reporting-menu/reportingmenu.controller.js b/plugins/CoreHome/angularjs/reporting-menu/reportingmenu.controller.js
index 1c94136827..d284034cd6 100644
--- a/plugins/CoreHome/angularjs/reporting-menu/reportingmenu.controller.js
+++ b/plugins/CoreHome/angularjs/reporting-menu/reportingmenu.controller.js
@@ -133,6 +133,12 @@
                     }
                 }
             });
+            if ('object' === typeof widgetsHelper && widgetsHelper.availableWidgets) {
+                // lets also update widgetslist so will be easier to update list of available widgets in dashboard selector
+                // immediately
+                delete widgetsHelper.availableWidgets;
+                widgetsHelper.getAvailableWidgets();
+            }
         });
 
         $rootScope.$on('$locationChangeSuccess', function () {
diff --git a/plugins/CoreHome/lang/en.json b/plugins/CoreHome/lang/en.json
index a013ab30ae..1677e05d28 100644
--- a/plugins/CoreHome/lang/en.json
+++ b/plugins/CoreHome/lang/en.json
@@ -29,6 +29,10 @@
         "InjectedHostSuperUserWarning": "Piwik may be misconfigured (for example, if Piwik was recently moved to a new server or URL). You can either %1$sclick here and add %2$s as the valid Piwik hostname (if you trust it)%3$s, or %4$sclick here and go to %5$s to access Piwik safely%6$s.",
         "InjectedHostWarningIntro": "You are now accessing Piwik from %1$s, but Piwik has been configured to run at this address: %2$s.",
         "JavascriptDisabled": "JavaScript must be enabled in order for you to use Piwik in standard view.<br \/>However, it seems JavaScript is either disabled or not supported by your browser.<br \/>To use standard view, enable JavaScript by changing your browser options, then %1$stry again%2$s.<br \/>",
+        "VisitStatusOrdered": "Ordered",
+        "VisitStatusOrderedThenAbandoned": "Ordered then Abandoned Cart",
+        "VisitTypeReturning": "Returning",
+        "VisitTypeReturningCustomer": "Returning Customer",
         "MainNavigation": "Main navigation",
         "MakeOneTimeDonation": "Make a one time donation, instead.",
         "Menu": "Menu",
diff --git a/plugins/CoreHome/templates/_dataTableActions.twig b/plugins/CoreHome/templates/_dataTableActions.twig
index dca99c47c7..537fdffd97 100644
--- a/plugins/CoreHome/templates/_dataTableActions.twig
+++ b/plugins/CoreHome/templates/_dataTableActions.twig
@@ -46,9 +46,11 @@
             {{ visualizationIcons|raw }}
         {% endif %}
 
+        {% if properties.show_export %}
         <a class='dropdown-button dataTableAction activateExportSelection'
            href='javascript:;' title="{{ 'General_ExportThisReport'|translate|e('html_attr') }}"
            data-activates='dropdownExport{{ randomIdForDropdown }}'><span class="icon-export"></span></a>
+        {% endif %}
 
         {% if isPluginLoaded('Annotations') and not properties.hide_annotations_view %}
             <a class='dataTableAction annotationView'
@@ -68,6 +70,7 @@
             </a>
         {% endif %}
 
+        {% if properties.show_export %}
         <ul id='dropdownExport{{ randomIdForDropdown }}' class='dropdown-content exportToFormatItems'>
             {% set requestParams = properties.request_parameters_to_modify|json_encode %}
             <li><a target="_blank" requestParams="{{ requestParams|e('html_attr') }}" methodToCall="{{ properties.apiMethodToRequestDataTable }}" format="CSV" filter_limit="{{ properties.filter_limit }}">CSV</a></li>
@@ -89,6 +92,8 @@
                 </li>
             {% endif %}
         </ul>
+        {% endif %}
+
         <ul id='dropdownConfigure{{ randomIdForDropdown }}' class='dropdown-content tableConfiguration'>
             {% if properties.show_flatten_table %}
                 {% if clientSideParameters.flat is defined and clientSideParameters.flat == 1 %}