- changing ways of applying filters to a datatable, now $table->filter('Piwik_DataTable_Filter_Sort', array ('nb_visits', 'desc'));

- refs #640 now accurately reporting nb_uniq_visitors in all reports
- adding tests

11 files changed, 154 insertions, 61 deletions

diff --git a/core/Archive.php b/core/Archive.php
index f6ca63c2b6..a07d0ef9d4 100644
--- a/core/Archive.php
+++ b/core/Archive.php
@@ -55,6 +55,7 @@ abstract class Piwik_Archive
 	const INDEX_NB_CONVERSIONS = 8;
 	const INDEX_REVENUE = 9;
 	const INDEX_GOALS = 10;
+	const INDEX_SUM_DAILY_NB_UNIQ_VISITORS = 11;
 	
 	const INDEX_GOAL_NB_CONVERSIONS = 1;
 	const INDEX_GOAL_REVENUE = 2;
@@ -70,6 +71,7 @@ abstract class Piwik_Archive
 				Piwik_Archive::INDEX_NB_CONVERSIONS 		=> 'nb_conversions',
 				Piwik_Archive::INDEX_REVENUE				=> 'revenue',
 				Piwik_Archive::INDEX_GOALS					=> 'goals',
+				Piwik_Archive::INDEX_SUM_DAILY_NB_UNIQ_VISITORS => 'sum_daily_nb_uniq_visitors',
 			);
 
 	public static $mappingFromIdToNameGoal = array(
@@ -91,6 +93,7 @@ abstract class Piwik_Archive
 				'nb_conversions' 			=> Piwik_Archive::INDEX_NB_CONVERSIONS,
 				'revenue' 					=> Piwik_Archive::INDEX_REVENUE,
 				'goals'						=> Piwik_Archive::INDEX_GOALS,
+				'sum_daily_nb_uniq_visitors' => Piwik_Archive::INDEX_SUM_DAILY_NB_UNIQ_VISITORS,
 	);
 	
 	/**
diff --git a/core/ArchiveProcessing/Period.php b/core/ArchiveProcessing/Period.php
index 2a5fae2e93..d1e60db76e 100644
--- a/core/ArchiveProcessing/Period.php
+++ b/core/ArchiveProcessing/Period.php
@@ -21,6 +21,14 @@
  */
 class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 {
+	/*
+	 * Array of (column name before => column name renamed) of the columns for which sum operation is invalid. 
+	 * The summed value is not accurate and these columns will be renamed accordingly.
+	 */
+	static public $invalidSummedColumnNameToRenamedName = array(
+		Piwik_Archive::INDEX_NB_UNIQ_VISITORS => Piwik_Archive::INDEX_SUM_DAILY_NB_UNIQ_VISITORS 
+	);
+	
 	public function __construct()
 	{
 		parent::__construct();
@@ -148,6 +156,7 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 	 * 				)
 	 */
 	public function archiveDataTable(	$aRecordName, 
+										$invalidSummedColumnNameToRenamedName = null,
 										$maximumRowsInDataTableLevelZero = null, 
 										$maximumRowsInSubDataTable = null,
 										$columnToSortByBeforeTruncation = null )
@@ -160,7 +169,7 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 		$nameToCount = array();
 		foreach($aRecordName as $recordName)
 		{
-			$table = $this->getRecordDataTableSum($recordName);
+			$table = $this->getRecordDataTableSum($recordName, $invalidSummedColumnNameToRenamedName);
 			
 			$nameToCount[$recordName]['level0'] =  $table->getRowsCount();
 			$nameToCount[$recordName]['recursive'] =  $table->getRowsCountRecursive();
@@ -180,9 +189,10 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 	 * The resulting DataTable is returned.
 	 *
 	 * @param string $name
+	 * @param array columns in the array (old name, new name) to be renamed as the sum operation is not valid on them (eg. nb_uniq_visitors->sum_daily_nb_uniq_visitors)
 	 * @return Piwik_DataTable
 	 */
-	protected function getRecordDataTableSum( $name )
+	protected function getRecordDataTableSum( $name, $invalidSummedColumnNameToRenamedName )
 	{
 		$table = new Piwik_DataTable;
 		foreach($this->archives as $archive)
@@ -193,6 +203,15 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 			$table->addDataTable($datatableToSum);
 			$archive->freeBlob($name);
 		}
+		
+		if(is_null($invalidSummedColumnNameToRenamedName))
+		{
+			$invalidSummedColumnNameToRenamedName = self::$invalidSummedColumnNameToRenamedName;
+		}
+		foreach($invalidSummedColumnNameToRenamedName as $oldName => $newName)
+		{
+			$table->renameColumn($oldName, $newName);
+		}
 		return $table;
 	}
 	
diff --git a/core/DataTable.php b/core/DataTable.php
index b195d64480..5a4b727380 100644
--- a/core/DataTable.php
+++ b/core/DataTable.php
@@ -214,7 +214,7 @@ class Piwik_DataTable
 
 	const ID_SUMMARY_ROW = -1;
 	const LABEL_SUMMARY_ROW = -1;
-
+	
 	/**
 	 * Maximum nesting level
 	 * 
@@ -320,6 +320,23 @@ class Piwik_DataTable
 	}
 
 	/**
+	 * Apply a filter to this datatable
+	 * 
+	 * @param $className eg. Piwik_DataTable_Filter_Sort
+	 * @param $parameters eg. array('nb_visits', 'asc')
+	 */
+	public function filter( $className, $parameters = array() )
+	{
+		$reflectionObj = new ReflectionClass($className);
+		
+		// the first parameter of a filter is the DataTable
+		// we add the current datatable as the parameter
+		$parameters = array_merge(array($this), $parameters);
+		
+		$filter = $reflectionObj->newInstanceArgs($parameters); 
+	}
+	
+	/**
 	 * Queue a DataTable_Filter that will be applied when applyQueuedFilters() is called.
 	 * (just before sending the datatable back to the browser (or API, etc.)
 	 *
@@ -349,13 +366,7 @@ class Piwik_DataTable
 				$this->setRowsCountBeforeLimitFilter();
 			}
 			
-			$reflectionObj = new ReflectionClass($filter['className']);
-			
-			// the first parameter of a filter is the DataTable
-			// we add the current datatable as the parameter
-			$filter['parameters'] = array_merge(array($this), $filter['parameters']);
-			
-			$filter = $reflectionObj->newInstanceArgs($filter['parameters']); 
+			$this->filter($filter['className'], $filter['parameters']);
 		}
 		$this->queuedFilters = array();
 	}
@@ -649,16 +660,57 @@ class Piwik_DataTable
 	 */
 	public function deleteColumn( $name )
 	{
+		$this->deleteColumns(array($name));
+	}
+
+	/**
+	 * Rename a column in all rows
+	 * @param $oldName
+	 * @param $newName
+	 */
+	public function renameColumn( $oldName, $newName )
+	{
 		foreach($this->getRows() as $row)
 		{
-			$row->deleteColumn($name);
+			$row->renameColumn($oldName, $newName);
+			if(($idSubDataTable = $row->getIdSubDataTable()) !== null)
+			{
+				Piwik_DataTable_Manager::getInstance()->getTable($idSubDataTable)->renameColumn($oldName, $newName);
+			}
 		}
 		if(!is_null($this->summaryRow))
+		{			
+			$this->summaryRow->renameColumn($oldName, $newName);
+		}
+	}
+	
+	/**
+	 * Delete columns by name in all rows
+	 *
+	 * @param string $name
+	 */
+	public function deleteColumns($names, $deleteRecursiveInSubtables = false)
+	{
+		foreach($this->getRows() as $row)
 		{
-			$this->summaryRow->deleteColumn($name);
+			foreach($names as $name)
+			{
+				$row->deleteColumn($name);
+			}
+			if(($idSubDataTable = $row->getIdSubDataTable()) !== null)
+			{
+				Piwik_DataTable_Manager::getInstance()->getTable($idSubDataTable)->deleteColumns($names, $deleteRecursiveInSubtables);
+			}
+		}
+		if(!is_null($this->summaryRow))
+		{			
+			foreach($names as $name)
+			{
+				$this->summaryRow->deleteColumn($name);
+			}
 		}
 	}
-
+	
 	/**
 	 * Deletes the ith row 
 	 *
@@ -826,11 +878,16 @@ class Piwik_DataTable
 		
 		if($depth > self::MAXIMUM_DEPTH_LEVEL_ALLOWED)
 		{
+			$depth = 0;
 			throw new Exception("Maximum recursion level of ".self::MAXIMUM_DEPTH_LEVEL_ALLOWED. " reached. You have probably set a DataTable_Row with an associated DataTable which belongs already to its parent hierarchy.");
 		}
 		if( !is_null($maximumRowsInDataTable) )
 		{
-			$filter = new Piwik_DataTable_Filter_AddSummaryRow($this, $maximumRowsInDataTable - 1, Piwik_DataTable::LABEL_SUMMARY_ROW, $columnToSortByBeforeTruncation);
+			$this->filter('Piwik_DataTable_Filter_AddSummaryRow', 
+							array(	$maximumRowsInDataTable - 1, 
+									Piwik_DataTable::LABEL_SUMMARY_ROW, 
+									$columnToSortByBeforeTruncation)
+					);
 		}
 		
 		// For each row, get the serialized row
@@ -1056,13 +1113,8 @@ class Piwik_DataTable
 		$cleanRow = array();
 		foreach($array as $label => $row)
 		{
-			// TODO I think this requirement is not true anymore:
-			// we make sure that the label column is first in the list! 
-			// important for the UI javascript mainly... 
-			
-			// array_merge doesn't work here as it reindex the numeric value
-			// see the test testMergeArray in PHP_Related.test.php
 			$cleanRow[Piwik_DataTable_Row::DATATABLE_ASSOCIATED] = null;
+			// we put the 'label' column first as it looks prettier in API results
 			$cleanRow[Piwik_DataTable_Row::COLUMNS] = array('label' => $label) + $row;
 			if(!is_null($subtablePerLabel)
 				// some rows of this table don't have subtables 
diff --git a/core/DataTable/Array.php b/core/DataTable/Array.php
index 891b47ff1b..63bbd6704b 100644
--- a/core/DataTable/Array.php
+++ b/core/DataTable/Array.php
@@ -105,6 +105,19 @@ class Piwik_DataTable_Array
 	}
 	
 	/**
+	 * Apply a filter to all tables in the array
+	 * @param $className
+	 * @param $parameters
+	 */
+	public function filter($className, $parameters = array())
+	{
+		foreach($this->array as $table)
+		{
+			$table->filter($className, $parameters);
+		}
+	}
+	
+	/**
 	 * Returns the array of DataTable
 	 *
 	 * @return array of Piwik_DataTable
diff --git a/core/DataTable/Filter/AddSummaryRow.php b/core/DataTable/Filter/AddSummaryRow.php
index e98133f026..95b90198e9 100644
--- a/core/DataTable/Filter/AddSummaryRow.php
+++ b/core/DataTable/Filter/AddSummaryRow.php
@@ -45,9 +45,8 @@ class Piwik_DataTable_Filter_AddSummaryRow extends Piwik_DataTable_Filter
 
 	protected function filter()
 	{
-		$filter = new Piwik_DataTable_Filter_Sort($this->table, 
-													$this->columnToSortByBeforeTruncating, 
-													'desc');
+		$this->table->filter('Piwik_DataTable_Filter_Sort', 
+							array( $this->columnToSortByBeforeTruncating, 'desc'));
 		
 		$rows = $this->table->getRows();
 		$count = $this->table->getRowsCount();
@@ -67,7 +66,7 @@ class Piwik_DataTable_Filter_AddSummaryRow extends Piwik_DataTable_Filter
 		}
 		
 		$newRow->addColumn('label', $this->labelSummaryRow);
-		$filter = new Piwik_DataTable_Filter_Limit($this->table, 0, $this->startRowToSummarize);
+		$this->table->filter('Piwik_DataTable_Filter_Limit', array(0, $this->startRowToSummarize));
 		$this->table->addSummaryRow($newRow);
 	}
 }
diff --git a/core/DataTable/Filter/ExcludeLowPopulation.php b/core/DataTable/Filter/ExcludeLowPopulation.php
index 94acd0baea..8479d24202 100644
--- a/core/DataTable/Filter/ExcludeLowPopulation.php
+++ b/core/DataTable/Filter/ExcludeLowPopulation.php
@@ -22,32 +22,33 @@
 class Piwik_DataTable_Filter_ExcludeLowPopulation extends Piwik_DataTable_Filter
 {
 	static public $minimumValue;
-	public function __construct( $table, $columnToFilter, $minimumValue )
+	const MINIMUM_SIGNIFICANT_PERCENTAGE_THRESHOLD = 0.02;
+	public function __construct( $table, $columnToFilter, $minimumValue, $minimumPercentageThreshold = false )
 	{
 		parent::__construct($table);
 		$this->columnToFilter = $columnToFilter;
 		
 		if($minimumValue == 0)
 		{
-			$minimumPercentageThreshold = 0.02;
+			if($minimumPercentageThreshold === false)
+			{
+				$minimumPercentageThreshold = self::MINIMUM_SIGNIFICANT_PERCENTAGE_THRESHOLD;
+			}
 			$allValues = $this->table->getColumn($this->columnToFilter);
 			$sumValues = array_sum($allValues);
 			$minimumValue = $sumValues * $minimumPercentageThreshold;
 		}
 		self::$minimumValue = $minimumValue;
-		if(self::$minimumValue > 1)
-		{
-			$this->filter();
-		}
+		$this->filter();
 	}
 	
 	function filter()
 	{
-		$filter = new Piwik_DataTable_Filter_ColumnCallbackDeleteRow(
-												$this->table, 
-												$this->columnToFilter, 
-												array("Piwik_DataTable_Filter_ExcludeLowPopulation", "excludeLowPopulation")
-											);
+		$this->table->filter('Piwik_DataTable_Filter_ColumnCallbackDeleteRow',
+							array($this->columnToFilter, 
+								array("Piwik_DataTable_Filter_ExcludeLowPopulation", "excludeLowPopulation")
+							)
+						);
 	}
 	
 	static public function excludeLowPopulation($value)
diff --git a/core/DataTable/Row.php b/core/DataTable/Row.php
index 2a26327b38..d29b70f5e1 100644
--- a/core/DataTable/Row.php
+++ b/core/DataTable/Row.php
@@ -19,9 +19,12 @@
  * - idSubtable: a row can be linked to a SubTable
  * 
  * IMPORTANT: Make sure that the column named 'label' contains at least one non-numeric character.
- * Otherwise the method addDataTable() or sumRow() would fail because they would consider
- * the 'label' as being a numeric column to sum.
+ *            Otherwise the method addDataTable() or sumRow() would fail because they would consider
+ *            the 'label' as being a numeric column to sum.
  * 
+ * PERFORMANCE: Do *not* add new fields except if necessary in this object. New fields will be 
+ *              serialized and recorded in the DB millions of times. This object size is critical and must be under control.
+ *              
  * @package Piwik_DataTable
  * @subpackage Piwik_DataTable_Row
  * 
@@ -139,6 +142,15 @@ class Piwik_DataTable_Row
 		return true;
 	}
 	
+	public function renameColumn($oldName, $newName)
+	{
+		if(isset($this->c[self::COLUMNS][$oldName]))
+		{
+			$this->c[self::COLUMNS][$newName] = $this->c[self::COLUMNS][$oldName];
+			unset($this->c[self::COLUMNS][$oldName]);
+		}
+	}
+	
 	/**
 	 * Returns the given column
 	 * 
@@ -308,14 +320,6 @@ class Piwik_DataTable_Row
 		}
 		$this->c[self::METADATA][$name] = $value;
 	}
-	
-	//TODO this should not be hardcoded here
-	protected $columnsExcludedFromSum = array(
-		'label' => true, 
-		'nb_uniq_visitors' => true, 
-		'entry_nb_uniq_visitors' => true, 
-		'exit_nb_uniq_visitors' => true,
-	);
 
 	/**
 	 * Sums the given $row columns values to the existing row' columns values.
@@ -330,7 +334,7 @@ class Piwik_DataTable_Row
 	{
 		foreach($rowToSum->getColumns() as $columnToSumName => $columnToSumValue)
 		{
-			if(!isset($this->columnsExcludedFromSum[$columnToSumName]))
+			if($columnToSumName != 'label')
 			{
 				$thisColumnValue = $this->getColumn($columnToSumName);
 				$newValue = $this->sumRowArray($thisColumnValue, $columnToSumValue);
diff --git a/core/FrontController.php b/core/FrontController.php
index 7e804f0938..bccd477520 100644
--- a/core/FrontController.php
+++ b/core/FrontController.php
@@ -299,6 +299,8 @@ class Exception_PluginDeactivated extends Exception
 	}
 }
 
+
+// for more information see http://dev.piwik.org/trac/ticket/374
 function destroy(&$var) 
 {
 	if (is_object($var)) $var->__destruct();
diff --git a/core/ViewDataTable/GenerateGraphData.php b/core/ViewDataTable/GenerateGraphData.php
index d259d9e76b..1267a728c8 100644
--- a/core/ViewDataTable/GenerateGraphData.php
+++ b/core/ViewDataTable/GenerateGraphData.php
@@ -66,8 +66,8 @@ abstract class Piwik_ViewDataTable_GenerateGraphData extends Piwik_ViewDataTable
 		}
 		$this->mainAlreadyExecuted = true;
 	
-		// we load the data with the filters applied
-		$this->disableGenericFilters();
+		// the queued filters will be manually applied later. This is to ensure that filtering using search
+		// will be done on the table before the labels are enhanced (see ReplaceColumnNames)
 		$this->disableQueuedFilters();
 		$this->loadDataTableFromAPI();
 		
@@ -75,10 +75,12 @@ abstract class Piwik_ViewDataTable_GenerateGraphData extends Piwik_ViewDataTable
 		if(!empty($graphLimit))
 		{
 			$offsetStartSummary = $this->getGraphLimit() - 1;
-			$filter = new Piwik_DataTable_Filter_AddSummaryRow($this->dataTable, 
-																$offsetStartSummary, 
-																Piwik_Translate('General_Others'), 
-																Piwik_Archive::INDEX_NB_VISITS);
+			$this->dataTable->filter('Piwik_DataTable_Filter_AddSummaryRow', 
+										array($offsetStartSummary, 
+										Piwik_Translate('General_Others'), 
+										Piwik_Archive::INDEX_NB_VISITS
+										)
+									);
 		}
 		$this->dataAvailable = $this->dataTable->getRowsCount() != 0;
 		
@@ -104,11 +106,9 @@ abstract class Piwik_ViewDataTable_GenerateGraphData extends Piwik_ViewDataTable
 	{
 		$this->dataTable->applyQueuedFilters();
 		// We apply a filter to the DataTable, decoding the label column (useful for keywords for example)
-		$filter = new Piwik_DataTable_Filter_ColumnCallbackReplace(
-									$this->dataTable, 
-									'label', 
-									'urldecode'
-								);
+		$this->dataTable->filter('Piwik_DataTable_Filter_ColumnCallbackReplace',
+									array('label','urldecode')
+							);
 		$data = array();
 		foreach($this->dataTable->getRows() as $row)
 		{
diff --git a/core/ViewDataTable/HtmlTable/AllColumns.php b/core/ViewDataTable/HtmlTable/AllColumns.php
index 2a83c9ae80..59327d7071 100644
--- a/core/ViewDataTable/HtmlTable/AllColumns.php
+++ b/core/ViewDataTable/HtmlTable/AllColumns.php
@@ -29,8 +29,8 @@ class Piwik_ViewDataTable_HtmlTable_AllColumns extends Piwik_ViewDataTable_HtmlT
 										'nb_actions_per_visit', 
 										'avg_time_on_site', 
 										'bounce_rate'));
-		$filter = new Piwik_DataTable_Filter_ColumnCallbackReplace($this->dataTable, 'avg_time_on_site', create_function('$averageTimeOnSite', 'return Piwik::getPrettyTimeFromSeconds($averageTimeOnSite);'));
-		$filter = new Piwik_DataTable_Filter_ColumnCallbackReplace($this->dataTable, 'bounce_rate', create_function('$bounceRate', 'return $bounceRate."%";'));
+		$this->dataTable->filter('Piwik_DataTable_Filter_ColumnCallbackReplace', array('avg_time_on_site', create_function('$averageTimeOnSite', 'return Piwik::getPrettyTimeFromSeconds($averageTimeOnSite);')));
+		$this->dataTable->filter('Piwik_DataTable_Filter_ColumnCallbackReplace', array('bounce_rate', create_function('$bounceRate', 'return $bounceRate."%";')));
 		$this->setColumnTranslation('nb_actions_per_visit', Piwik_Translate('General_ColumnActionsPerVisit'));
 		$this->setColumnTranslation('avg_time_on_site', Piwik_Translate('General_ColumnAvgTimeOnSite'));
 		$this->setColumnTranslation('bounce_rate', Piwik_Translate('General_ColumnBounceRate'));
diff --git a/core/ViewDataTable/HtmlTable/Goals.php b/core/ViewDataTable/HtmlTable/Goals.php
index 6711f7602c..cf025bc173 100644
--- a/core/ViewDataTable/HtmlTable/Goals.php
+++ b/core/ViewDataTable/HtmlTable/Goals.php
@@ -75,8 +75,8 @@ class Piwik_ViewDataTable_HtmlTable_Goals extends Piwik_ViewDataTable_HtmlTable
 		$this->columnsToPercentageFilter[] = 'goals_conversion_rate';
 		foreach($this->columnsToPercentageFilter as $columnName)
 		{
-			$filter = new Piwik_DataTable_Filter_ColumnCallbackReplace($this->dataTable, $columnName, create_function('$rate', 'return $rate."%";'));
+			$this->dataTable->filter('Piwik_DataTable_Filter_ColumnCallbackReplace', array($columnName, create_function('$rate', 'return $rate."%";')));
 		}
-		$filter = new Piwik_DataTable_Filter_ColumnCallbackReplace($this->dataTable, 'revenue_per_visit', array("Piwik", "getPrettyMoney"));
+		$this->dataTable->filter('Piwik_DataTable_Filter_ColumnCallbackReplace', array('revenue_per_visit', array("Piwik", "getPrettyMoney")));
 	}
 }