Show flattened columns as extra columns in UI (#12524)

* Show each dimension as seperate column in html tables for flattened reports

* set default columns

* rename first dimension column for flattened reports to label

* modify rows right before rendering to prevent plugins to overwrite the displayed columns again

* Use datatable config to allow splitting dimensions into separate columns

* Hide datatable option if report has only one dimension

* ensure to add only needed metric dimensions

* adds ui test

* reorder datatable options

* Add additional columns before generic filters are applied to make sorting of those columns possible

* Improve table layout with multipe label columns

* show logo only in first column

* Add new metadata so the original combine label is still available for row actions

* improve check for multiple lables

* show row action always in last label column

* updates UI file

* fix Others handling for flattened reports

* ensure label translation is set correctly

* show concatenated dimension names as flatten report label

* search combined labels in flattened tables showing dimensions separately

* more ui tests

* fix handling of summary row for concatenated dimensions

* updates expected test files

* fix insights layout

* fix realtime visits table layout

* fix ui tests

* ensure dimensions have correct display order

15 files changed, 174 insertions, 20 deletions

diff --git a/plugins/Actions/tests/UI/ActionsDataTable_spec.js b/plugins/Actions/tests/UI/ActionsDataTable_spec.js
index b09a55eb7c..9c9f855544 100644
--- a/plugins/Actions/tests/UI/ActionsDataTable_spec.js
+++ b/plugins/Actions/tests/UI/ActionsDataTable_spec.js
@@ -25,10 +25,10 @@ describe("ActionsDataTable", function () {
     });
 
     it("should load subtables correctly when row clicked", async function() {
-        firstRow = await page.jQuery('tr.subDataTable:first');
-        await firstRow.click();
         secondRow = await page.jQuery('tr.subDataTable:eq(2)');
         await secondRow.click();
+        firstRow = await page.jQuery('tr.subDataTable:first');
+        await firstRow.click();
         await page.mouse.move(-10, -10);
 
         await page.waitForNetworkIdle();
diff --git a/plugins/CoreHome/CoreHome.php b/plugins/CoreHome/CoreHome.php
index 79fac09408..3c80765c77 100644
--- a/plugins/CoreHome/CoreHome.php
+++ b/plugins/CoreHome/CoreHome.php
@@ -327,6 +327,8 @@ class CoreHome extends \Piwik\Plugin
         $translationKeys[] = 'CoreHome_ExcludeRowsWithLowPopulation';
         $translationKeys[] = 'CoreHome_DataTableIncludeAggregateRows';
         $translationKeys[] = 'CoreHome_DataTableExcludeAggregateRows';
+        $translationKeys[] = 'CoreHome_DataTableCombineDimensions';
+        $translationKeys[] = 'CoreHome_DataTableShowDimensions';
         $translationKeys[] = 'CoreHome_Default';
         $translationKeys[] = 'CoreHome_FormatMetrics';
         $translationKeys[] = 'CoreHome_ShowExportUrl';
diff --git a/plugins/CoreHome/javascripts/dataTable.js b/plugins/CoreHome/javascripts/dataTable.js
index 6ee9e93600..74be5d8a10 100644
--- a/plugins/CoreHome/javascripts/dataTable.js
+++ b/plugins/CoreHome/javascripts/dataTable.js
@@ -252,8 +252,21 @@ $.extend(DataTable.prototype, UIControl.prototype, {
 
         var params = {};
         for (var key in self.param) {
-            if (typeof self.param[key] != "undefined" && self.param[key] != '')
+            if (typeof self.param[key] != "undefined" && self.param[key] != '') {
+                if (key == 'filter_column' || key == 'filter_column_recursive' ) {
+                    // search in (metadata) `combinedLabel` when dimensions are shown separately in flattened tables
+                    // needs to be overwritten for each request as switching a searched table might return no results
+                    // otherwise, as search column doesn't fit anymore
+                    if (self.param.flat == "1" && self.param.show_dimensions == "1") {
+                        params[key] = 'combinedLabel';
+                    } else {
+                        params[key] = 'label';
+                    }
+                    continue;
+                }
+
                 params[key] = self.param[key];
+            }
         }
 
         ajaxRequest.addParams(params, 'get');
@@ -456,7 +469,7 @@ $.extend(DataTable.prototype, UIControl.prototype, {
                 labelWidth = maxLabelWidth; // prevent for instance table in Actions-Pages is not too wide
             }
 
-            return parseInt(labelWidth, 10);
+            return parseInt(labelWidth / $('tr:nth-child(1) td.label', domElem).length, 10);
         }
 
         function getLabelColumnMinWidth(domElem)
@@ -1322,6 +1335,13 @@ $.extend(DataTable.prototype, UIControl.prototype, {
                 }
             }));
 
+        $('.dataTableShowDimensions', domElem)
+            .each(function () {
+                setText(this, 'show_dimensions', 'CoreHome_DataTableCombineDimensions',
+                    'CoreHome_DataTableShowDimensions');
+            })
+            .click(generateClickCallback('show_dimensions'));
+
         // handle pivot by
         $('.dataTablePivotBySubtable', domElem)
             .each(function () {
@@ -1437,6 +1457,16 @@ $.extend(DataTable.prototype, UIControl.prototype, {
         // label (first column of a data row) or not
         $("th:first-child", domElem).addClass('label');
         $("td:first-child", domElem).addClass('label');
+
+        var metadata = this.getReportMetadata();
+
+        if (self.param.flat == "1" && self.param.show_dimensions == "1" && metadata.dimensions && Object.keys(metadata.dimensions).length > 1) {
+            for (var i = 1; i < Object.keys(metadata.dimensions).length; i++) {
+                $("th:nth-child("+(i+1)+")", domElem).addClass('label');
+                $("td:nth-child("+(i+1)+")", domElem).addClass('label');
+            }
+        }
+
         $("tr td", domElem).addClass('column');
     },
 
@@ -1754,7 +1784,9 @@ $.extend(DataTable.prototype, UIControl.prototype, {
         var details = _pk_translate('General_LearnMore', [' (<a href="https://matomo.org/faq/how-to/faq_54/" rel="noreferrer noopener" target="_blank">', '</a>)']);
 
         domElem.find('tr.summaryRow').each(function () {
-            var labelSpan = $(this).find('.label .value');
+            var labelSpan = $(this).find('.label .value').filter(function(index, elem){
+                return $(elem).text() != '-';
+            }).last();
             var defaultLabel = labelSpan.text();
 
             $(this).hover(function() {
@@ -1789,7 +1821,7 @@ $.extend(DataTable.prototype, UIControl.prototype, {
 
         trs.each(function () {
             var tr = $(this);
-            var td = tr.find('td:first');
+            var td = tr.find('td.label:last');
 
             // call initTr on all actions that are available for the report
             for (var i = 0; i < availableActionsForReport.length; i++) {
@@ -1918,7 +1950,7 @@ $.extend(DataTable.prototype, UIControl.prototype, {
             return;
         }
 
-        var td = tr.find('td:first');
+        var td = tr.find('td.label:last');
         var actions = tr.find('div.dataTableRowActions');
 
         if (!actions) {
diff --git a/plugins/CoreHome/javascripts/dataTable_rowactions.js b/plugins/CoreHome/javascripts/dataTable_rowactions.js
index 5d51e06302..9d8751293b 100644
--- a/plugins/CoreHome/javascripts/dataTable_rowactions.js
+++ b/plugins/CoreHome/javascripts/dataTable_rowactions.js
@@ -198,6 +198,12 @@ DataTable_RowAction.prototype.trigger = function (tr, e, subTableLabel) {
 
 /** Get the label string from a tr dom element */
 DataTable_RowAction.prototype.getLabelFromTr = function (tr) {
+    var rowMetadata = this.getRowMetadata(tr);
+
+    if (rowMetadata.combinedLabel) {
+        return '@' + rowMetadata.combinedLabel;
+    }
+
     var label = tr.find('span.label');
 
     // handle truncation
diff --git a/plugins/CoreHome/lang/en.json b/plugins/CoreHome/lang/en.json
index cd302097eb..1c6970ce51 100644
--- a/plugins/CoreHome/lang/en.json
+++ b/plugins/CoreHome/lang/en.json
@@ -14,6 +14,8 @@
         "DataTableExcludeAggregateRows": "Aggregate rows are shown %s Hide them",
         "DataTableIncludeAggregateRows": "Aggregate rows are hidden %s Show them",
         "DataTableHowToSearch": "Press enter or click the search icon to search",
+        "DataTableShowDimensions": "Dimensions are combined %s Show dimensions separately",
+        "DataTableCombineDimensions": "Dimensions are shown separately %s Show dimensions combined",
         "Default": "default",
         "DonateCall1": "Matomo will always cost you nothing to use, but that doesn't mean it costs us nothing to make.",
         "DonateCall2": "Matomo needs your continued support to grow and thrive.",
diff --git a/plugins/CoreHome/templates/_dataTableActions.twig b/plugins/CoreHome/templates/_dataTableActions.twig
index 095c5a0aee..4266644440 100644
--- a/plugins/CoreHome/templates/_dataTableActions.twig
+++ b/plugins/CoreHome/templates/_dataTableActions.twig
@@ -121,14 +121,19 @@
 
         <ul id='dropdownConfigure{{ randomIdForDropdown }}' class='dropdown-content tableConfiguration'>
             {% if properties.show_flatten_table %}
+                <li>
+                    <div class="configItem dataTableFlatten"></div>
+                </li>
                 {% if clientSideParameters.flat is defined and clientSideParameters.flat == 1 %}
+                    {% if hasMultipleDimensions|default %}
+                        <li>
+                            <div class="configItem dataTableShowDimensions"></div>
+                        </li>
+                    {% endif %}
                     <li>
                         <div class="configItem dataTableIncludeAggregateRows"></div>
                     </li>
                 {% endif %}
-                <li>
-                    <div class="configItem dataTableFlatten"></div>
-                </li>
             {% endif %}
             {% if not isDataTableEmpty and properties.show_totals_row|default(0) %}
                 <li>
diff --git a/plugins/CoreHome/templates/_dataTableCell.twig b/plugins/CoreHome/templates/_dataTableCell.twig
index a1b1c2c59a..7aca38defd 100644
--- a/plugins/CoreHome/templates/_dataTableCell.twig
+++ b/plugins/CoreHome/templates/_dataTableCell.twig
@@ -37,13 +37,13 @@
           title="{{ reportRatioTooltip|raw }} {{ totalRatioTooltip|e('html_attr') }}"
           >&nbsp;{{ rowPercentage }}</span>
 {%- endif %}
-
-{% if column=='label' %}
+{% set dimensions = dataTable.getMetadata('dimensions')|default([]) %}
+{% if column=='label' or column in dimensions %}
     {% import 'macros.twig' as piwik %}
 
     <span class='label{% if row.getMetadata('is_aggregate') %} highlighted{% endif %}'
     {% if properties is defined and properties.tooltip_metadata_name is not empty %}title="{{ row.getMetadata(properties.tooltip_metadata_name) }}"{% endif %}>
-        {{ piwik.logoHtml(row.getMetadata(), row.getColumn('label')) }}
+        {% if column=='label' %}{{ piwik.logoHtml(row.getMetadata(), row.getColumn('label')) }}{% endif %}
         {% if row.getMetadata('html_label_prefix') %}<span class='label-prefix'>{{ row.getMetadata('html_label_prefix') | raw }}&nbsp;</span>{% endif -%}
 {% endif %}<span class="value">
     {%- if row.getColumn(column) or (column=='label' and row.getColumn(column) is same as("0")) %}{% if column=='label' %}{{- row.getColumn(column)|rawSafeDecoded -}}{% else %}{% if row.getMetadata('html_column_' ~ column ~ '_prefix') %}<span class='column-prefix'>{{ row.getMetadata('html_column_' ~ column ~ '_prefix') | raw }}</span>{% endif -%}{{- row.getColumn(column)|number(2,0)|rawSafeDecoded -}}{% if row.getMetadata('html_column_' ~ column ~ '_suffix') %}<span class='column-suffix'>{{ row.getMetadata('html_column_' ~ column ~ '_suffix') | raw }}</span>{% endif -%}{% endif %}
diff --git a/plugins/CoreVisualizations/Visualizations/HtmlTable.php b/plugins/CoreVisualizations/Visualizations/HtmlTable.php
index 2fa6e6674b..a305c12538 100644
--- a/plugins/CoreVisualizations/Visualizations/HtmlTable.php
+++ b/plugins/CoreVisualizations/Visualizations/HtmlTable.php
@@ -9,9 +9,11 @@
 namespace Piwik\Plugins\CoreVisualizations\Visualizations;
 
 use Piwik\API\Request as ApiRequest;
+use Piwik\Columns\Dimension;
 use Piwik\Common;
 use Piwik\DataTable\Row;
 use Piwik\Metrics;
+use Piwik\DataTable;
 use Piwik\Period;
 use Piwik\Plugin\Visualization;
 
@@ -93,6 +95,55 @@ class HtmlTable extends Visualization
         if ($this->isPivoted()) {
             $this->config->columns_to_display = $this->dataTable->getColumns();
         }
+
+        // Note: This needs to be done right before rendering, as otherwise some plugins might change the columns to display again
+        if ($this->isFlattened()) {
+            $dimensions = $this->dataTable->getMetadata('dimensions');
+
+            $hasMultipleDimensions = is_array($dimensions) && count($dimensions) > 1;
+            $this->assignTemplateVar('hasMultipleDimensions', $hasMultipleDimensions);
+
+            if ($hasMultipleDimensions) {
+                if ($this->config->show_dimensions) {
+                    // ensure first metric translation is used as label if other dimensions are in separate columns
+                    $this->config->addTranslation('label', $this->config->translations[reset($dimensions)]);
+                } else {
+                    // concatenate dimensions if table is shown flattened
+                    foreach ($dimensions as $dimension) {
+                        $labels[] = $this->config->translations[$dimension];
+                    }
+                    $this->config->addTranslation('label', implode(' - ', $labels));
+                }
+            }
+
+            if ($this->config->show_dimensions && $hasMultipleDimensions) {
+
+
+                $properties = $this->config;
+                array_shift($dimensions); // shift away first dimension, as that will be shown as label
+
+                $this->dataTable->filter(function (DataTable $dataTable) use ($properties, $dimensions) {
+                    if (empty($properties->columns_to_display)) {
+                        $columns           = $dataTable->getColumns();
+                        $hasNbVisits       = in_array('nb_visits', $columns);
+                        $hasNbUniqVisitors = in_array('nb_uniq_visitors', $columns);
+
+                        $properties->setDefaultColumnsToDisplay($columns, $hasNbVisits, $hasNbUniqVisitors);
+                    }
+
+                    $label = array_search('label', $properties->columns_to_display);
+                    if ($label !== false) {
+                        unset($properties->columns_to_display[$label]);
+                    }
+
+                    foreach (array_reverse($dimensions) as $dimension) {
+                        array_unshift($properties->columns_to_display, $dimension);
+                    }
+
+                    array_unshift($properties->columns_to_display, 'label');
+                });
+            }
+        }
     }
 
     public function beforeGenericFiltersAreAppliedToLoadedDataTable()
@@ -104,6 +155,44 @@ class HtmlTable extends Visualization
         }
 
         parent::beforeGenericFiltersAreAppliedToLoadedDataTable();
+
+        // Note: This needs to be done right before generic filter are applied, to make sorting such columns possible
+        if ($this->isFlattened()) {
+            $dimensions = $this->dataTable->getMetadata('dimensions');
+
+            $hasMultipleDimensions = is_array($dimensions) && count($dimensions) > 1;
+
+            if ($hasMultipleDimensions) {
+                foreach (Dimension::getAllDimensions() as $dimension) {
+                    $dimensionId = str_replace('.', '_', $dimension->getId());
+                    $dimensionName = $dimension->getName();
+
+                    if (!empty($dimensionId) && !empty($dimensionName) && in_array($dimensionId, $dimensions)) {
+                        $this->config->translations[$dimensionId] = $dimensionName;
+                    }
+                }
+            }
+
+
+            if ($this->config->show_dimensions && $hasMultipleDimensions) {
+
+                $this->dataTable->filter(function($dataTable) use ($dimensions) {
+                    /** @var DataTable $dataTable */
+                    $rows = $dataTable->getRows();
+                    foreach ($rows as $row) {
+                        foreach ($dimensions as $dimension) {
+                            $row->setColumn($dimension, $row->getMetadata($dimension));
+                        }
+                    }
+                });
+
+                # replace original label column with first dimension
+                $firstDimension = array_shift($dimensions);
+                $this->dataTable->filter('ColumnCallbackAddMetadata', array('label', 'combinedLabel', function ($label) { return $label; }));
+                $this->dataTable->filter('ColumnDelete', array('label'));
+                $this->dataTable->filter('ReplaceColumnNames', array(array($firstDimension => 'label')));
+            }
+        }
     }
 
     protected function isPivoted()
@@ -122,4 +211,9 @@ class HtmlTable extends Visualization
     {
         return null;
     }
+
+    protected function isFlattened()
+    {
+        return $this->requestConfig->flat || Common::getRequestVar('flat', '');
+    }
 }
diff --git a/plugins/CoreVisualizations/Visualizations/HtmlTable/AllColumns.php b/plugins/CoreVisualizations/Visualizations/HtmlTable/AllColumns.php
index 9f754eccc4..d3221acb8f 100644
--- a/plugins/CoreVisualizations/Visualizations/HtmlTable/AllColumns.php
+++ b/plugins/CoreVisualizations/Visualizations/HtmlTable/AllColumns.php
@@ -63,6 +63,8 @@ class AllColumns extends HtmlTable
 
             $properties->columns_to_display = $columnsToDisplay;
         });
+
+        parent::beforeGenericFiltersAreAppliedToLoadedDataTable();
     }
 
     public function beforeLoadDataTable()
diff --git a/plugins/CoreVisualizations/Visualizations/HtmlTable/Config.php b/plugins/CoreVisualizations/Visualizations/HtmlTable/Config.php
index ba39a66c0f..94de8f18b3 100644
--- a/plugins/CoreVisualizations/Visualizations/HtmlTable/Config.php
+++ b/plugins/CoreVisualizations/Visualizations/HtmlTable/Config.php
@@ -9,6 +9,7 @@
 
 namespace Piwik\Plugins\CoreVisualizations\Visualizations\HtmlTable;
 
+use Piwik\Columns\Dimension;
 use Piwik\ViewDataTable\Config as VisualizationConfig;
 
 /**
@@ -100,6 +101,12 @@ class Config extends VisualizationConfig
      */
     public $report_ratio_columns = array();
 
+    /**
+     * If true, dimensions in flattened reports will be shown as separate columns
+     * @var bool
+     */
+    public $show_dimensions = false;
+
     public function __construct()
     {
         parent::__construct();
@@ -115,6 +122,7 @@ class Config extends VisualizationConfig
             'enable_sort',
             'keep_summary_row',
             'subtable_controller_action',
+            'show_dimensions',
         ));
 
         $this->addPropertiesThatCanBeOverwrittenByQueryParams(array(
@@ -126,6 +134,7 @@ class Config extends VisualizationConfig
             'disable_subtable_when_show_goals',
             'keep_summary_row',
             'highlight_summary_row',
+            'show_dimensions',
         ));
     }
 
diff --git a/plugins/CoreVisualizations/Visualizations/HtmlTable/RequestConfig.php b/plugins/CoreVisualizations/Visualizations/HtmlTable/RequestConfig.php
index dde538219c..804568bf08 100644
--- a/plugins/CoreVisualizations/Visualizations/HtmlTable/RequestConfig.php
+++ b/plugins/CoreVisualizations/Visualizations/HtmlTable/RequestConfig.php
@@ -58,12 +58,14 @@ class RequestConfig extends VisualizationRequestConfig
             'filter_sort_column',
             'filter_sort_order',
             'keep_summary_row',
-            'keep_totals_row'
+            'keep_totals_row',
+            'show_dimensions'
         ));
 
         $this->addPropertiesThatCanBeOverwrittenByQueryParams(array(
             'keep_summary_row',
             'keep_totals_row',
+            'show_dimensions',
         ));
     }
 
diff --git a/plugins/CoreVisualizations/templates/_dataTableViz_htmlTable.twig b/plugins/CoreVisualizations/templates/_dataTableViz_htmlTable.twig
index 10c28ff40b..c4b4659e03 100644
--- a/plugins/CoreVisualizations/templates/_dataTableViz_htmlTable.twig
+++ b/plugins/CoreVisualizations/templates/_dataTableViz_htmlTable.twig
@@ -21,7 +21,7 @@
             {%- for rowId, row in dataTable.getRows() -%}
                 {%- set rowHasSubtable = not subtablesAreDisabled and row.getIdSubDataTable() and properties.subtable_controller_action is not null -%}
                 {%- set rowSubtableId = row.getMetadata('idsubdatatable_in_db')|default(row.getIdSubDataTable()) -%}
-                {%- set isSummaryRow = rowId == constant('Piwik\\DataTable::ID_SUMMARY_ROW') -%}
+                {%- set isSummaryRow = rowId == constant('Piwik\\DataTable::ID_SUMMARY_ROW') or row.getMetadata('is_summary') -%}
                 {%- set shouldHighlightRow = isSummaryRow and properties.highlight_summary_row -%}
 
                 {# display this row if it doesn't have a subtable or if we don't replace the row with the subtable #}
diff --git a/plugins/Insights/templates/table_header.twig b/plugins/Insights/templates/table_header.twig
index 00dc29736a..bb71c41bd3 100644
--- a/plugins/Insights/templates/table_header.twig
+++ b/plugins/Insights/templates/table_header.twig
@@ -2,11 +2,11 @@
     <th class="label first">
         {{ metadata.reportName }}
     </th>
-    <th class="label orderBy {% if 'absolute' == properties.order_by %}active{% endif %}"
+    <th class="orderBy {% if 'absolute' == properties.order_by %}active{% endif %}"
         name="orderBy" value="absolute">
         {{ metadata.metricName }}
     </th>
-    <th class="label last orderBy {% if 'relative' == properties.order_by %}active{% endif %}"
+    <th class="last orderBy {% if 'relative' == properties.order_by %}active{% endif %}"
         name="orderBy" value="relative">
         {{ 'MultiSites_Evolution'|translate }}
     </th>
diff --git a/plugins/Live/templates/_totalVisitors.twig b/plugins/Live/templates/_totalVisitors.twig
index a00ff2b3eb..665617e926 100644
--- a/plugins/Live/templates/_totalVisitors.twig
+++ b/plugins/Live/templates/_totalVisitors.twig
@@ -5,10 +5,10 @@
             <th id="label" class="sortable label first" style="cursor: auto;">
                 <div id="thDIV">{{ 'General_Date'|translate }}</div>
             </th>
-            <th id="label" class="sortable label" style="cursor: auto;">
+            <th class="sortable" style="cursor: auto;">
                 <div id="thDIV">{{ 'General_ColumnNbVisits'|translate }}</div>
             </th>
-            <th id="label" class="sortable label" style="cursor: auto;">
+            <th class="sortable" style="cursor: auto;">
                 <div id="thDIV">{{ 'General_Actions'|translate }}</div>
             </th>
         </tr>
diff --git a/plugins/Morpheus/stylesheets/main.less b/plugins/Morpheus/stylesheets/main.less
index 61146031cc..e643ee193c 100644
--- a/plugins/Morpheus/stylesheets/main.less
+++ b/plugins/Morpheus/stylesheets/main.less
@@ -322,7 +322,7 @@ table.dataTable {
         padding-bottom: 16px;
         vertical-align: middle;
 
-        &:not(.first) {
+        &:not(.label) {
           text-align: right;
           padding-left: 28px;
           padding-right: 12px;