- memory and speed optimizations of archiving, refs #374 (hopefully fixing it but not sure yet)

- added support for profiling memory & time in Piwik via the events mechanism 
- small other changes

18 files changed, 148 insertions, 77 deletions

diff --git a/core/Archive/Single.php b/core/Archive/Single.php
index 4035e2804b..79003c3b62 100644
--- a/core/Archive/Single.php
+++ b/core/Archive/Single.php
@@ -303,7 +303,7 @@ class Piwik_Archive_Single extends Piwik_Archive
 	public function freeBlob( $name )
 	{
 		$this->blobCached[$name] = null; 
-//		$this->blobCached = array(); 
+		unset($this->blobCached[$name]);
 	}
 	
 	/**
@@ -452,7 +452,7 @@ class Piwik_Archive_Single extends Piwik_Archive
 		$this->preFetchBlob($name);
 		$dataTableToLoad = $this->getDataTable($name, $idSubTable);
 		$this->loadSubDataTables($name, $dataTableToLoad, $addMetadataSubtableId = true);
+		$this->freeBlob($name);
 		return $dataTableToLoad;		
 	}
 }
-?>
diff --git a/core/ArchiveProcessing.php b/core/ArchiveProcessing.php
index a728c98003..7bad89b1f2 100644
--- a/core/ArchiveProcessing.php
+++ b/core/ArchiveProcessing.php
@@ -174,13 +174,10 @@ abstract class Piwik_ArchiveProcessing
 	protected $debugAlwaysArchive = false;
 	
 	/**
-	 * Builds the archive processing object, 
-	 * Reads some configuration value from the config file
-	 *
+	 * Constructor
 	 */
 	public function __construct()
 	{
-		$this->debugAlwaysArchive = Zend_Registry::get('config')->Debug->always_archive_data;
 	}
 	
 	/**
@@ -480,6 +477,7 @@ abstract class Piwik_ArchiveProcessing
 		}
 		$record = new Piwik_ArchiveProcessing_Record_Blob($name, $value);
 		$this->insertRecord($record);
+		unset($record);
 		return true;
 	}
 	
@@ -491,7 +489,7 @@ abstract class Piwik_ArchiveProcessing
 	protected function insertRecord($record)
 	{
 		// table to use to save the data
-		if(Piwik::isNumeric($record->value))
+		if(is_numeric($record->value))
 		{
 			$table = $this->tableArchiveNumeric;
 		}
diff --git a/core/ArchiveProcessing/Day.php b/core/ArchiveProcessing/Day.php
index 77558a3f34..66782c9838 100644
--- a/core/ArchiveProcessing/Day.php
+++ b/core/ArchiveProcessing/Day.php
@@ -36,6 +36,7 @@ class Piwik_ArchiveProcessing_Day extends Piwik_ArchiveProcessing
 	{
 		parent::__construct();
 		$this->db = Zend_Registry::get('db');
+		$this->debugAlwaysArchive = Zend_Registry::get('config')->Debug->always_archive_data_day;
 	}
 	
 	/**
diff --git a/core/ArchiveProcessing/Period.php b/core/ArchiveProcessing/Period.php
index 3b13a97551..2a5fae2e93 100644
--- a/core/ArchiveProcessing/Period.php
+++ b/core/ArchiveProcessing/Period.php
@@ -21,6 +21,12 @@
  */
 class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 {
+	public function __construct()
+	{
+		parent::__construct();
+		$this->debugAlwaysArchive = Zend_Registry::get('config')->Debug->always_archive_data_period;
+	}
+	
 	/**
 	 * Sums all values for the given field names $aNames over the period
 	 * See @archiveNumericValuesGeneral for more information
@@ -160,11 +166,14 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 			$nameToCount[$recordName]['recursive'] =  $table->getRowsCountRecursive();
 			
 			$blob = $table->getSerialized( $maximumRowsInDataTableLevelZero, $maximumRowsInSubDataTable, $columnToSortByBeforeTruncation );
+			destroy($table);
 			$this->insertBlobRecord($recordName, $blob);
 		}
+		Piwik_DataTable_Manager::getInstance()->deleteAll();
+		
 		return $nameToCount;
 	}
-	
+
 	/**
 	 * This method selects all DataTables that have the name $name over the period.
 	 * It calls the appropriate methods that sum all these tables together.
diff --git a/core/ArchiveProcessing/Record.php b/core/ArchiveProcessing/Record.php
index cce1b9a0fc..c34c37552e 100644
--- a/core/ArchiveProcessing/Record.php
+++ b/core/ArchiveProcessing/Record.php
@@ -34,9 +34,6 @@ abstract class Piwik_ArchiveProcessing_Record
 		$this->value = $value;
 	}
 	
-	public function __destruct()
-	{
-	}
 }
 
 
diff --git a/core/DataTable.php b/core/DataTable.php
index 132a940d64..82dc72b74a 100644
--- a/core/DataTable.php
+++ b/core/DataTable.php
@@ -167,7 +167,17 @@ class Piwik_DataTable
 	 * 
 	 * @var bool
 	 */
-	protected $indexNotUpToDate = false;
+	protected $indexNotUpToDate = true;
+	
+	/**
+	 * This flag sets the index to be rebuild whenever a new row is added, 
+	 * as opposed to re-building the full index when getRowFromLabel is called.
+	 * This is to optimize and not rebuild the full Index in the case where we
+	 * add row, getRowFromLabel, addRow, getRowFromLabel thousands of times.
+	 * 
+	 * @var bool
+	 */
+	protected $rebuildIndexContinuously = false;
 	
 	/**
 	 * Column name of last time the table was sorted
@@ -222,6 +232,22 @@ class Piwik_DataTable
 	}
 
 	/**
+	 * At destruction we free all memory
+	 */
+	public function __destruct()
+	{
+		// destruct can be called several times
+		if(isset($this->rows))
+		{
+			foreach($this->getRows() as $row) {
+				destroy($row);
+			}
+			unset($this->rows);
+			Piwik_DataTable_Manager::getInstance()->setTableDeleted($this->getId());	
+		}
+	}
+	
+	/**
 	 * Sort the dataTable rows using the php callback function 
 	 *
 	 * @param string $functionCallback
@@ -351,7 +377,7 @@ class Piwik_DataTable
 			{
 				if( $labelToLookFor === self::LABEL_SUMMARY_ROW )
 				{
-					$this->addSummaryRow($row );
+					$this->addSummaryRow( $row );
 				}
 				else
 				{
@@ -382,6 +408,7 @@ class Piwik_DataTable
 	 */
 	public function getRowFromLabel( $label )
 	{
+		$this->rebuildIndexContinuously = true;
 		if($this->indexNotUpToDate)
 		{
 			$this->rebuildIndex();
@@ -411,7 +438,6 @@ class Piwik_DataTable
 		foreach($this->rows as $id => $row)
 		{
 			$label = $row->getColumn('label');
-		
 			if($label !== false)
 			{
 				$this->rowsIndexByLabel[$label] = $id;
@@ -441,14 +467,23 @@ class Piwik_DataTable
 	}
 
 	/**
-	 * Shortcut function used for performance reasons
+	 * Add a row to the table and rebuild the index if necessary
 	 * 
 	 * @param Piwik_DataTable_Row $row to add at the end of the array
 	 */
 	public function addRow( Piwik_DataTable_Row $row )
 	{
-		$this->rows[] = $row;	
-		$this->indexNotUpToDate = true;
+		$this->rows[] = $row;
+		if(!$this->indexNotUpToDate
+			&& $this->rebuildIndexContinuously)
+		{
+			$label = $row->getColumn('label');
+			if($label !== false)
+			{
+				$this->rowsIndexByLabel[$label] = count($this->rows)-1;
+			}
+			$this->indexNotUpToDate = false;
+		}
 	}
 
 	/**
@@ -1037,13 +1072,5 @@ class Piwik_DataTable
 		}
 	}
 
-	/**
-	 * At destruction we try to free memory
-	 * But php doesn't give us much control on this
-	 */
-	public function __destruct()
-	{
-		unset($this->rows);
-	}
-	
 }
+
diff --git a/core/DataTable/Filter/Sort.php b/core/DataTable/Filter/Sort.php
index 5e4e324c7e..222a2ca5d8 100644
--- a/core/DataTable/Filter/Sort.php
+++ b/core/DataTable/Filter/Sort.php
@@ -148,7 +148,7 @@ class Piwik_DataTable_Filter_Sort extends Piwik_DataTable_Filter
 		$this->columnToSort = $this->selectColumnToSort($row);
 		
 		$value = $row->getColumn($this->columnToSort);
-		if( Piwik::isNumeric($value))
+		if( is_numeric($value))
 		{
 			$methodToUse = "sort";
 		}
diff --git a/core/DataTable/Manager.php b/core/DataTable/Manager.php
index 50ee38e486..ea34b4e5d2 100644
--- a/core/DataTable/Manager.php
+++ b/core/DataTable/Manager.php
@@ -85,12 +85,17 @@ class Piwik_DataTable_Manager
 	 */
 	public function deleteAll()
 	{
+		foreach($this->tables as $id => $table) 
+		{
+			destroy($table);
+		}
 		$this->tables = array();
 		$this->lastTableId = 0;
 	}
 	
 	/**
-	 * Deletes the datatable given its id
+	 * Deletes (unsets) the datatable given its id and removes it from the manager
+	 * Subsequent get for this table will fail
 	 *
 	 * @param int $id
 	 */
@@ -98,11 +103,22 @@ class Piwik_DataTable_Manager
 	{
 		if(isset($this->tables[$id]))
 		{
-			$this->tables[$id] = null;
+			destroy($this->tables[$id]);
+			$this->setTableDeleted($id);
 		}
 	}
 	
 	/**
+	 * Remove the table from the manager (table has already been unset)
+	 * @param $id
+	 * @return void
+	 */
+	public function setTableDeleted($id)
+	{
+		$this->tables[$id] = null;
+	}
+	
+	/**
 	 * Debug only. Dumps all tables currently registered in the Manager
 	 *
 	 * @return void
diff --git a/core/DataTable/Row.php b/core/DataTable/Row.php
index 20d46c9a50..2a26327b38 100644
--- a/core/DataTable/Row.php
+++ b/core/DataTable/Row.php
@@ -80,7 +80,7 @@ class Piwik_DataTable_Row
 			&& $row[self::DATATABLE_ASSOCIATED] instanceof Piwik_DataTable)
 		{
 			$this->c[self::DATATABLE_ASSOCIATED] = $row[self::DATATABLE_ASSOCIATED]->getId();
-		}
+		}	
 	}
 	
 	/**
@@ -273,16 +273,7 @@ class Piwik_DataTable_Row
 	 */
 	public function setColumn($name, $value)
 	{
-		if(isset($this->c[self::COLUMNS][$name])
-			|| $name != 'label')
-		{
-			$this->c[self::COLUMNS][$name] = $value;
-		}
-		// we make sure when adding the label it goes first in the table
-		else
-		{
-			$this->c[self::COLUMNS] = array($name => $value) + $this->c[self::COLUMNS];
-		}
+		$this->c[self::COLUMNS][$name] = $value;
 	}
 	
 	/**
@@ -318,6 +309,14 @@ class Piwik_DataTable_Row
 		$this->c[self::METADATA][$name] = $value;
 	}
 	
+	//TODO this should not be hardcoded here
+	protected $columnsExcludedFromSum = array(
+		'label' => true, 
+		'nb_uniq_visitors' => true, 
+		'entry_nb_uniq_visitors' => true, 
+		'exit_nb_uniq_visitors' => true,
+	);
+
 	/**
 	 * Sums the given $row columns values to the existing row' columns values.
 	 * It will sum only the int or float values of $row.
@@ -331,13 +330,7 @@ class Piwik_DataTable_Row
 	{
 		foreach($rowToSum->getColumns() as $columnToSumName => $columnToSumValue)
 		{
-			//TODO this should not be hardcoded here.
-			if($columnToSumName != 'label'
-				&& $columnToSumName != 'nb_uniq_visitors'
-				&& $columnToSumName != 'entry_nb_uniq_visitors'
-				&& $columnToSumName != 'exit_nb_uniq_visitors'
-				
-				)
+			if(!isset($this->columnsExcludedFromSum[$columnToSumName]))
 			{
 				$thisColumnValue = $this->getColumn($columnToSumName);
 				$newValue = $this->sumRowArray($thisColumnValue, $columnToSumValue);
@@ -348,36 +341,33 @@ class Piwik_DataTable_Row
 	
 	protected function sumRowArray( $thisColumnValue, $columnToSumValue )
 	{
-		$newValue = 0;
-		if(Piwik::isNumeric($columnToSumValue))
+		if(is_numeric($columnToSumValue))
 		{
 			if($thisColumnValue === false)
 			{
 				$thisColumnValue = 0;
 			}
-			$newValue = $thisColumnValue + $columnToSumValue;
+			return $thisColumnValue + $columnToSumValue;
 		}
-		elseif(is_array($columnToSumValue))
+		
+		if(is_array($columnToSumValue))
 		{
-			$newValue = array();
 			if($thisColumnValue == false)
 			{
-				$newValue = $columnToSumValue;
+				return $columnToSumValue;
 			}
-			else
+			$newValue = $thisColumnValue;
+			foreach($columnToSumValue as $arrayIndex => $arrayValue)
 			{
-				$newValue = $thisColumnValue;
-				foreach($columnToSumValue as $arrayIndex => $arrayValue)
+				if(!isset($newValue[$arrayIndex])) 
 				{
-					if(!isset($newValue[$arrayIndex])) 
-					{
-						$newValue[$arrayIndex] = false;
-					}
-					$newValue[$arrayIndex] = $this->sumRowArray($newValue[$arrayIndex], $arrayValue);
+					$newValue[$arrayIndex] = false;
 				}
+				$newValue[$arrayIndex] = $this->sumRowArray($newValue[$arrayIndex], $arrayValue);
 			}
+			return $newValue;
 		}
-		return $newValue;
+		return 0;
 	}
 	
 	/**
diff --git a/core/FrontController.php b/core/FrontController.php
index aa2fc38a59..7e804f0938 100644
--- a/core/FrontController.php
+++ b/core/FrontController.php
@@ -298,3 +298,10 @@ class Exception_PluginDeactivated extends Exception
 		parent::__construct("The plugin '$module' is not activated. You can activate the plugin on the 'Plugins admin' page.");
 	}
 }
+
+function destroy(&$var) 
+{
+	if (is_object($var)) $var->__destruct();
+	unset($var);
+	$var = null;
+}
diff --git a/core/Log.php b/core/Log.php
index 23cb476cbf..25166a3258 100644
--- a/core/Log.php
+++ b/core/Log.php
@@ -93,7 +93,7 @@ abstract class Piwik_Log extends Zend_Log
 		if (empty($this->_writers)) {
 			throw new Zend_Log_Exception('No writers were added');
 		}
-
+		
 		$event['timestamp'] = date('c');
 
 		// pack into event required by filters and writers
@@ -115,12 +115,10 @@ abstract class Piwik_Log extends Zend_Log
 }
 
 /**
- * 
- * 
  * @package Piwik_Log
  */
 class Piwik_Log_Formatter_FileFormatter implements Zend_Log_Formatter_Interface
-{
+{	
 	/**
 	 * Formats data into a single line to be written by the writer.
 	 *
@@ -141,6 +139,12 @@ class Piwik_Log_Formatter_FileFormatter implements Zend_Log_Formatter_Interface
 
 class Piwik_Log_Formatter_ScreenFormatter implements Zend_Log_Formatter_Interface
 {
+	function formatEvent($event)
+	{
+		// no injection in error messages, backtrace when displayed on screen
+		return array_map('htmlspecialchars', $event);
+	}
+	
 	function format($string)
 	{
 		$string = self::getFormattedString($string);
diff --git a/core/Log/Error.php b/core/Log/Error.php
index c2ae08f9fd..c5db22cbbc 100644
--- a/core/Log/Error.php
+++ b/core/Log/Error.php
@@ -73,6 +73,8 @@ class Piwik_Log_Formatter_Error_ScreenFormatter extends Piwik_Log_Formatter_Scre
      */
     public function format($event)
     {
+    	$event = parent::formatEvent($event);
+    	
 		$errno = $event['errno'] ;
 		$errstr = $event['message'] ;
 		$errfile = $event['errfile'] ;
diff --git a/core/Log/Exception.php b/core/Log/Exception.php
index 5845bae4f0..4e678cc211 100644
--- a/core/Log/Exception.php
+++ b/core/Log/Exception.php
@@ -73,6 +73,7 @@ class Piwik_Log_Formatter_Exception_ScreenFormatter extends Piwik_Log_Formatter_
 	 */
 	public function format($event)
 	{
+    	$event = parent::formatEvent($event);
 		$errno = $event['errno'] ;
 		$errstr = $event['message'] ;
 		$errfile = $event['errfile'] ;
diff --git a/core/Log/Message.php b/core/Log/Message.php
index 37e44a1a24..648d1dc57c 100644
--- a/core/Log/Message.php
+++ b/core/Log/Message.php
@@ -38,7 +38,6 @@ class Piwik_Log_Message extends Piwik_Log
 	{
 		$event = array();
 		$event['message'] = $message;
-		
 		parent::log($event);
 	}
 }
diff --git a/core/Piwik.php b/core/Piwik.php
index e987523f9e..bc49c0051e 100644
--- a/core/Piwik.php
+++ b/core/Piwik.php
@@ -334,6 +334,13 @@ class Piwik
 	{
 		echo Zend_Registry::get('timer');
 	}
+
+	static public function printMemoryLeak($prefix = '', $suffix = '<br>')
+	{
+		echo $prefix;
+		echo Zend_Registry::get('timer')->getMemoryLeak();
+		echo $suffix;
+	}
 	
 	static public function printMemoryUsage( $prefixString = null )
 	{
@@ -384,11 +391,6 @@ class Piwik
 		return in_array(substr(php_sapi_name(), 0, 3), array('cgi', 'cli'));
 	}
 	
-	static public function isNumeric($value)
-	{
-		return is_numeric($value);
-	}
-	
 	static public function getCurrency()
 	{
 		static $symbol = null;
diff --git a/core/PluginsManager.php b/core/PluginsManager.php
index ffdb24e784..35c934fad0 100644
--- a/core/PluginsManager.php
+++ b/core/PluginsManager.php
@@ -495,7 +495,8 @@ class Piwik_Plugin_Exception extends Exception
  */
 function Piwik_PostEvent( $eventName,  &$object = null, $info = array() )
 {
-	Piwik_PluginsManager::getInstance()->dispatcher->post( $object, $eventName, $info, true, false );
+	$notification = new Piwik_Event_Notification($object, $eventName, $info);
+	Piwik_PluginsManager::getInstance()->dispatcher->postNotification( $notification, true, false );
 }
 
 /**
@@ -505,3 +506,20 @@ function Piwik_AddAction( $hookName, $function )
 {
 	Piwik_PluginsManager::getInstance()->dispatcher->addObserver( $function, $hookName );
 }
+
+class Piwik_Event_Notification extends Event_Notification
+{
+	static $showProfiler = false;
+	function increaseNotificationCount($className, $method) {
+		parent::increaseNotificationCount();
+		if(self::$showProfiler)
+		{
+			echo "after $className -> $method <br>";
+			echo "-"; Piwik::printTimer();
+			echo "<br>";
+			echo "-"; Piwik::printMemoryLeak();
+			echo "<br>";
+		}
+	}
+}
+
diff --git a/core/Tracker.php b/core/Tracker.php
index 2dfaa9ef3c..97cd1519e3 100644
--- a/core/Tracker.php
+++ b/core/Tracker.php
@@ -119,7 +119,7 @@ class Piwik_Tracker
 		}
 		printDebug("End of the page.");
 		
-		if($GLOBALS['DEBUGPIWIK'] === true)
+		if($GLOBALS['PIWIK_TRACKER_DEBUG'] === true)
 		{
 			Piwik::printSqlProfilingReportTracker(self::$db);
 		}
@@ -203,7 +203,7 @@ class Piwik_Tracker
 	
 	protected function outputTransparentGif()
 	{
-		if( !isset($GLOBALS['DEBUGPIWIK']) || !$GLOBALS['DEBUGPIWIK'] ) 
+		if( !isset($GLOBALS['PIWIK_TRACKER_DEBUG']) || !$GLOBALS['PIWIK_TRACKER_DEBUG'] ) 
 		{
 			$trans_gif_64 = "R0lGODlhAQABAIAAAAAAAAAAACH5BAEAAAAALAAAAAABAAEAAAICRAEAOw==";
 			header("Content-type: image/gif");
@@ -310,7 +310,7 @@ class Piwik_Tracker
 
 function printDebug( $info = '' )
 {
-	if(isset($GLOBALS['DEBUGPIWIK']) && $GLOBALS['DEBUGPIWIK'])
+	if(isset($GLOBALS['PIWIK_TRACKER_DEBUG']) && $GLOBALS['PIWIK_TRACKER_DEBUG'])
 	{
 		if(is_array($info))
 		{
diff --git a/core/Tracker/Visit.php b/core/Tracker/Visit.php
index bb34e5843c..0a54946d84 100644
--- a/core/Tracker/Visit.php
+++ b/core/Tracker/Visit.php
@@ -101,7 +101,7 @@ class Piwik_Tracker_Visit implements Piwik_Tracker_Visit_Interface
 		}
 		$actionId = $action->getIdAction();
 
-		if($GLOBALS['DEBUGPIWIK'])
+		if($GLOBALS['PIWIK_TRACKER_DEBUG'])
 		{
 			switch($action->getActionType()) {
 				case Piwik_Tracker_Action::TYPE_ACTION: