Ensure number of rows and row metadata equals in processed reports (#12650)

* Ensure number of rows and row metadata equals so they can be matched by ID

* reset $rowsMetadata to empty DataTable if no row had metadata

* update test files

9 files changed, 80 insertions, 59 deletions

diff --git a/plugins/API/ProcessedReport.php b/plugins/API/ProcessedReport.php
index 1cd327c8c7..6caac672c3 100644
--- a/plugins/API/ProcessedReport.php
+++ b/plugins/API/ProcessedReport.php
@@ -617,6 +617,8 @@ class ProcessedReport
 
         $formatter = new Formatter();
 
+        $hasNonEmptyRowData = false;
+
         foreach ($simpleDataTable->getRows() as $row) {
             $rowMetrics = $row->getColumns();
 
@@ -668,10 +670,12 @@ class ProcessedReport
                 $rowMetadata = $row->getMetadata();
                 $idSubDataTable = $row->getIdSubDataTable();
 
-                // Create a row metadata only if there are metadata to insert
+                // always add a metadata row - even if empty, so the number of rows and metadata are equal and can be matched directly
+                $metadataRow = new Row();
+                $rowsMetadata->addRow($metadataRow);
+
                 if (count($rowMetadata) > 0 || !is_null($idSubDataTable)) {
-                    $metadataRow = new Row();
-                    $rowsMetadata->addRow($metadataRow);
+                    $hasNonEmptyRowData = true;
 
                     foreach ($rowMetadata as $metadataKey => $metadataValue) {
                         $metadataRow->addColumn($metadataKey, $metadataValue);
@@ -684,6 +688,11 @@ class ProcessedReport
             }
         }
 
+        // reset $rowsMetadata to empty DataTable if no row had metadata
+        if ($hasNonEmptyRowData === false) {
+            $rowsMetadata = new DataTable();
+        }
+
         return array(
             $enhancedDataTable,
             $rowsMetadata
diff --git a/tests/PHPUnit/System/expected/test_CustomEvents_Events.getAction_lastN__API.getProcessedReport_day.xml b/tests/PHPUnit/System/expected/test_CustomEvents_Events.getAction_lastN__API.getProcessedReport_day.xml
index 78218dc7e3..caa8eb0452 100644
--- a/tests/PHPUnit/System/expected/test_CustomEvents_Events.getAction_lastN__API.getProcessedReport_day.xml
+++ b/tests/PHPUnit/System/expected/test_CustomEvents_Events.getAction_lastN__API.getProcessedReport_day.xml
@@ -282,57 +282,57 @@
 			<row>
 				
 				<segment>eventAction==playTrailer</segment>
-				<idsubdatatable>1564</idsubdatatable>
+				<idsubdatatable>1576</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==Search</segment>
-				<idsubdatatable>1557</idsubdatatable>
+				<idsubdatatable>1569</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==play25%25</segment>
-				<idsubdatatable>1559</idsubdatatable>
+				<idsubdatatable>1571</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==play50%25</segment>
-				<idsubdatatable>1560</idsubdatatable>
+				<idsubdatatable>1572</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==play75%25</segment>
-				<idsubdatatable>1561</idsubdatatable>
+				<idsubdatatable>1573</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==playEnd</segment>
-				<idsubdatatable>1562</idsubdatatable>
+				<idsubdatatable>1574</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==rating</segment>
-				<idsubdatatable>1563</idsubdatatable>
+				<idsubdatatable>1575</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==clickBuyNow</segment>
-				<idsubdatatable>1565</idsubdatatable>
+				<idsubdatatable>1577</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==event+action+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+---%26gt%3B+SHOULD+APPEAR+IN+TEST+OUTPUT+NOT+TRUNCATED+%26lt%3B---</segment>
-				<idsubdatatable>1567</idsubdatatable>
+				<idsubdatatable>1579</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==play</segment>
-				<idsubdatatable>1558</idsubdatatable>
+				<idsubdatatable>1570</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==playStart</segment>
-				<idsubdatatable>1566</idsubdatatable>
+				<idsubdatatable>1578</idsubdatatable>
 			</row>
 			<row>
 				<segment>eventAction==Purchase</segment>
@@ -345,57 +345,57 @@
 			<row>
 				
 				<segment>eventAction==playTrailer</segment>
-				<idsubdatatable>1576</idsubdatatable>
+				<idsubdatatable>1588</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==Search</segment>
-				<idsubdatatable>1569</idsubdatatable>
+				<idsubdatatable>1581</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==play25%25</segment>
-				<idsubdatatable>1571</idsubdatatable>
+				<idsubdatatable>1583</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==play50%25</segment>
-				<idsubdatatable>1572</idsubdatatable>
+				<idsubdatatable>1584</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==play75%25</segment>
-				<idsubdatatable>1573</idsubdatatable>
+				<idsubdatatable>1585</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==playEnd</segment>
-				<idsubdatatable>1574</idsubdatatable>
+				<idsubdatatable>1586</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==rating</segment>
-				<idsubdatatable>1575</idsubdatatable>
+				<idsubdatatable>1587</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==clickBuyNow</segment>
-				<idsubdatatable>1577</idsubdatatable>
+				<idsubdatatable>1589</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==event+action+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+---%26gt%3B+SHOULD+APPEAR+IN+TEST+OUTPUT+NOT+TRUNCATED+%26lt%3B---</segment>
-				<idsubdatatable>1579</idsubdatatable>
+				<idsubdatatable>1591</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==play</segment>
-				<idsubdatatable>1570</idsubdatatable>
+				<idsubdatatable>1582</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventAction==playStart</segment>
-				<idsubdatatable>1578</idsubdatatable>
+				<idsubdatatable>1590</idsubdatatable>
 			</row>
 			<row>
 				<segment>eventAction==Purchase</segment>
diff --git a/tests/PHPUnit/System/expected/test_CustomEvents_Events.getName_lastN__API.getProcessedReport_day.xml b/tests/PHPUnit/System/expected/test_CustomEvents_Events.getName_lastN__API.getProcessedReport_day.xml
index deb9083be8..6f99fa8bd0 100644
--- a/tests/PHPUnit/System/expected/test_CustomEvents_Events.getName_lastN__API.getProcessedReport_day.xml
+++ b/tests/PHPUnit/System/expected/test_CustomEvents_Events.getName_lastN__API.getProcessedReport_day.xml
@@ -183,72 +183,72 @@
 			<row>
 				
 				<segment>eventName==Spirited+Away+%28%E5%8D%83%E3%81%A8%E5%8D%83%E5%B0%8B%E3%81%AE%E7%A5%9E%E9%9A%A0%E3%81%97%29</segment>
-				<idsubdatatable>1652</idsubdatatable>
+				<idsubdatatable>1676</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventName==La+fianc%C3%A9e+de+l%26%23039%3Beau</segment>
-				<idsubdatatable>1651</idsubdatatable>
+				<idsubdatatable>1675</idsubdatatable>
 			</row>
 			<row>
 				
-				<idsubdatatable>1650</idsubdatatable>
+				<idsubdatatable>1674</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventName==event+name+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+---%26gt%3B+SHOULD+APPEAR+IN+TEST+OUTPUT+NOT+TRUNCATED+%26lt%3B---</segment>
-				<idsubdatatable>1656</idsubdatatable>
+				<idsubdatatable>1680</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventName==Ponyo+%28%E5%B4%96%E3%81%AE%E4%B8%8A%E3%81%AE%E3%83%9D%E3%83%8B%E3%83%A7%29</segment>
-				<idsubdatatable>1654</idsubdatatable>
+				<idsubdatatable>1678</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventName==Princess+Mononoke+%28%E3%82%82%E3%81%AE%E3%81%AE%E3%81%91%E5%A7%AB%29</segment>
-				<idsubdatatable>1653</idsubdatatable>
+				<idsubdatatable>1677</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventName==Search+query+here</segment>
-				<idsubdatatable>1655</idsubdatatable>
+				<idsubdatatable>1679</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Monday, January 4, 2010">
 			<row>
 				
 				<segment>eventName==Spirited+Away+%28%E5%8D%83%E3%81%A8%E5%8D%83%E5%B0%8B%E3%81%AE%E7%A5%9E%E9%9A%A0%E3%81%97%29</segment>
-				<idsubdatatable>1660</idsubdatatable>
+				<idsubdatatable>1684</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventName==La+fianc%C3%A9e+de+l%26%23039%3Beau</segment>
-				<idsubdatatable>1659</idsubdatatable>
+				<idsubdatatable>1683</idsubdatatable>
 			</row>
 			<row>
 				
-				<idsubdatatable>1658</idsubdatatable>
+				<idsubdatatable>1682</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventName==event+name+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+Extremely+long+---%26gt%3B+SHOULD+APPEAR+IN+TEST+OUTPUT+NOT+TRUNCATED+%26lt%3B---</segment>
-				<idsubdatatable>1664</idsubdatatable>
+				<idsubdatatable>1688</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventName==Ponyo+%28%E5%B4%96%E3%81%AE%E4%B8%8A%E3%81%AE%E3%83%9D%E3%83%8B%E3%83%A7%29</segment>
-				<idsubdatatable>1662</idsubdatatable>
+				<idsubdatatable>1686</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventName==Princess+Mononoke+%28%E3%82%82%E3%81%AE%E3%81%AE%E3%81%91%E5%A7%AB%29</segment>
-				<idsubdatatable>1661</idsubdatatable>
+				<idsubdatatable>1685</idsubdatatable>
 			</row>
 			<row>
 				
 				<segment>eventName==Search+query+here</segment>
-				<idsubdatatable>1663</idsubdatatable>
+				<idsubdatatable>1687</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Tuesday, January 5, 2010" />
diff --git a/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits_hideColumns___API.getProcessedReport_day.xml b/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits_hideColumns___API.getProcessedReport_day.xml
index 442db1a9b3..b0e2bb4254 100644
--- a/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits_hideColumns___API.getProcessedReport_day.xml
+++ b/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits_hideColumns___API.getProcessedReport_day.xml
@@ -60,6 +60,12 @@
 			
 			<idsubdatatable>5188</idsubdatatable>
 		</row>
+		<row>
+		</row>
+		<row>
+		</row>
+		<row>
+		</row>
 	</reportMetadata>
 	<reportTotal>
 		<entry_bounce_count>1</entry_bounce_count>
diff --git a/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits_showColumns___API.getProcessedReport_day.xml b/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits_showColumns___API.getProcessedReport_day.xml
index 33c6b4c63d..077ee6d63e 100644
--- a/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits_showColumns___API.getProcessedReport_day.xml
+++ b/tests/PHPUnit/System/expected/test_OneVisitorTwoVisits_showColumns___API.getProcessedReport_day.xml
@@ -69,6 +69,12 @@
 			
 			<idsubdatatable>5192</idsubdatatable>
 		</row>
+		<row>
+		</row>
+		<row>
+		</row>
+		<row>
+		</row>
 	</reportMetadata>
 	<reportTotal>
 		<nb_hits>4</nb_hits>
diff --git a/tests/PHPUnit/System/expected/test_SiteSearch_CustomVariables.getCustomVariables_firstSite_lastN__API.getProcessedReport_day.xml b/tests/PHPUnit/System/expected/test_SiteSearch_CustomVariables.getCustomVariables_firstSite_lastN__API.getProcessedReport_day.xml
index aa7ced88fc..732789475c 100644
--- a/tests/PHPUnit/System/expected/test_SiteSearch_CustomVariables.getCustomVariables_firstSite_lastN__API.getProcessedReport_day.xml
+++ b/tests/PHPUnit/System/expected/test_SiteSearch_CustomVariables.getCustomVariables_firstSite_lastN__API.getProcessedReport_day.xml
@@ -120,7 +120,7 @@
 				</slots>
 				
 				<segment>customVariableName==_pk_scount</segment>
-				<idsubdatatable>3237</idsubdatatable>
+				<idsubdatatable>3277</idsubdatatable>
 			</row>
 			<row>
 				<slots>
@@ -131,7 +131,7 @@
 				</slots>
 				
 				<segment>customVariableName==_pk_scat</segment>
-				<idsubdatatable>3236</idsubdatatable>
+				<idsubdatatable>3276</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Monday, January 4, 2010">
@@ -144,7 +144,7 @@
 				</slots>
 				
 				<segment>customVariableName==_pk_scount</segment>
-				<idsubdatatable>3240</idsubdatatable>
+				<idsubdatatable>3280</idsubdatatable>
 			</row>
 			<row>
 				<slots>
@@ -155,7 +155,7 @@
 				</slots>
 				
 				<segment>customVariableName==_pk_scat</segment>
-				<idsubdatatable>3239</idsubdatatable>
+				<idsubdatatable>3279</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Tuesday, January 5, 2010" />
diff --git a/tests/PHPUnit/System/expected/test_SiteSearch_CustomVariables.getCustomVariables_firstSite_lastN__API.getProcessedReport_month.xml b/tests/PHPUnit/System/expected/test_SiteSearch_CustomVariables.getCustomVariables_firstSite_lastN__API.getProcessedReport_month.xml
index 636bb3183d..c96accd28c 100644
--- a/tests/PHPUnit/System/expected/test_SiteSearch_CustomVariables.getCustomVariables_firstSite_lastN__API.getProcessedReport_month.xml
+++ b/tests/PHPUnit/System/expected/test_SiteSearch_CustomVariables.getCustomVariables_firstSite_lastN__API.getProcessedReport_month.xml
@@ -89,7 +89,7 @@
 				</slots>
 				
 				<segment>customVariableName==_pk_scount</segment>
-				<idsubdatatable>3262</idsubdatatable>
+				<idsubdatatable>3307</idsubdatatable>
 			</row>
 			<row>
 				<slots>
@@ -100,7 +100,7 @@
 				</slots>
 				
 				<segment>customVariableName==_pk_scat</segment>
-				<idsubdatatable>3261</idsubdatatable>
+				<idsubdatatable>3306</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="February 2010" />
diff --git a/tests/PHPUnit/System/expected/test_TwoVisitors_twoWebsites_differentDays_Actions.getPageTitles_firstSite_lastN__API.getProcessedReport_day.xml b/tests/PHPUnit/System/expected/test_TwoVisitors_twoWebsites_differentDays_Actions.getPageTitles_firstSite_lastN__API.getProcessedReport_day.xml
index 2ba4f93a54..2a60c771ec 100644
--- a/tests/PHPUnit/System/expected/test_TwoVisitors_twoWebsites_differentDays_Actions.getPageTitles_firstSite_lastN__API.getProcessedReport_day.xml
+++ b/tests/PHPUnit/System/expected/test_TwoVisitors_twoWebsites_differentDays_Actions.getPageTitles_firstSite_lastN__API.getProcessedReport_day.xml
@@ -184,51 +184,51 @@
 		<result prettyDate="Tuesday, January 5, 2010">
 			<row>
 				
-				<idsubdatatable>2014</idsubdatatable>
+				<idsubdatatable>2031</idsubdatatable>
 			</row>
 			<row>
 				
-				<idsubdatatable>2013</idsubdatatable>
+				<idsubdatatable>2030</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Wednesday, January 6, 2010">
 			<row>
 				
-				<idsubdatatable>2018</idsubdatatable>
+				<idsubdatatable>2035</idsubdatatable>
 			</row>
 			<row>
 				
-				<idsubdatatable>2017</idsubdatatable>
+				<idsubdatatable>2034</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Thursday, January 7, 2010">
 			<row>
 				
-				<idsubdatatable>2022</idsubdatatable>
+				<idsubdatatable>2039</idsubdatatable>
 			</row>
 			<row>
 				
-				<idsubdatatable>2021</idsubdatatable>
+				<idsubdatatable>2038</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Friday, January 8, 2010">
 			<row>
 				
-				<idsubdatatable>2026</idsubdatatable>
+				<idsubdatatable>2043</idsubdatatable>
 			</row>
 			<row>
 				
-				<idsubdatatable>2025</idsubdatatable>
+				<idsubdatatable>2042</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Saturday, January 9, 2010">
 			<row>
 				
-				<idsubdatatable>2030</idsubdatatable>
+				<idsubdatatable>2047</idsubdatatable>
 			</row>
 			<row>
 				
-				<idsubdatatable>2029</idsubdatatable>
+				<idsubdatatable>2046</idsubdatatable>
 			</row>
 		</result>
 	</reportMetadata>
diff --git a/tests/PHPUnit/System/expected/test_TwoVisitors_twoWebsites_differentDays_Referrers.getWebsites_firstSite_lastN__API.getProcessedReport_day.xml b/tests/PHPUnit/System/expected/test_TwoVisitors_twoWebsites_differentDays_Referrers.getWebsites_firstSite_lastN__API.getProcessedReport_day.xml
index 394432870a..918a8a608e 100644
--- a/tests/PHPUnit/System/expected/test_TwoVisitors_twoWebsites_differentDays_Referrers.getWebsites_firstSite_lastN__API.getProcessedReport_day.xml
+++ b/tests/PHPUnit/System/expected/test_TwoVisitors_twoWebsites_differentDays_Referrers.getWebsites_firstSite_lastN__API.getProcessedReport_day.xml
@@ -116,14 +116,14 @@
 			<row>
 				
 				<segment>referrerName==referrer.com</segment>
-				<idsubdatatable>1965</idsubdatatable>
+				<idsubdatatable>1979</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Monday, January 4, 2010">
 			<row>
 				
 				<segment>referrerName==referrer.com</segment>
-				<idsubdatatable>1967</idsubdatatable>
+				<idsubdatatable>1981</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Tuesday, January 5, 2010" />
@@ -133,14 +133,14 @@
 			<row>
 				
 				<segment>referrerName==referrer.com</segment>
-				<idsubdatatable>1972</idsubdatatable>
+				<idsubdatatable>1986</idsubdatatable>
 			</row>
 		</result>
 		<result prettyDate="Saturday, January 9, 2010">
 			<row>
 				
 				<segment>referrerName==referrer.com</segment>
-				<idsubdatatable>1974</idsubdatatable>
+				<idsubdatatable>1988</idsubdatatable>
 			</row>
 		</result>
 	</reportMetadata>