17 files changed, 122 insertions, 80 deletions

diff --git a/api_test.php b/api_test.php
new file mode 100644
index 0000000000..f7729fed09
--- /dev/null
+++ b/api_test.php
@@ -0,0 +1,17 @@
+Use cases
+
+A request returns 
+
+- one datatable
+- one datable with lots of filters applied
+- an HTML page containing 3 datatables + other values
+- 2 datatable being from different websites, different dates, different filters applied
+- the results of 3 other requests with the same parameters
+
+
+
+<?php
+
+
+
+?>
diff --git a/index.php b/index.php
index 35c33f5711..0de5c4354c 100755
--- a/index.php
+++ b/index.php
@@ -72,7 +72,7 @@ $doNotDrop = array(
 		Piwik::prefixTable('log_link_visit_action'),
 		Piwik::prefixTable('log_action'),
 		Piwik::prefixTable('log_profiling'),
-		Piwik::prefixTable('archive'),
+//		Piwik::prefixTable('archive'),
 	);
 
 Piwik::dropTables($doNotDrop);
@@ -104,34 +104,38 @@ Zend_Loader::loadClass('Piwik_Archive');
 Zend_Loader::loadClass('Piwik_Date');
 
 Piwik::printMemoryUsage('Before archiving');
-$test = new Piwik_Archive;
-$period = new Piwik_Period_Day( Piwik_Date::today() );
-$site = new Piwik_Site(1);
-$test->setPeriod($period);
-$test->setSite($site);
-
-
-$test = new Piwik_Archive;
-$period = new Piwik_Period_Month(Piwik_Date::today());
-$site = new Piwik_Site(1);
-$test->setPeriod($period);
-$test->setSite($site);
-Piwik::log("visits=".$test->get('nb_visits'));
-Piwik::log("max_actions=".$test->get('max_actions'));
-Piwik::log("UserSettings_resolution = ".$test->getDataTable('UserSettings_resolution'));
+//$test = new Piwik_Archive;
+//$period = new Piwik_Period_Day( Piwik_Date::today() );
+//$site = new Piwik_Site(1);
+//$test->setPeriod($period);
+//$test->setSite($site);
+//
+//
+//$test = new Piwik_Archive;
+//$period = new Piwik_Period_Month(Piwik_Date::today());
+//$site = new Piwik_Site(1);
+//$test->setPeriod($period);
+//$test->setSite($site);
+//Piwik::log("visits=".$test->get('nb_visits'));
+//Piwik::log("max_actions=".$test->get('max_actions'));
+//Piwik::log("UserSettings_resolution = ".$test->getDataTable('UserSettings_resolution'));
 
 
 $test = new Piwik_Archive;
+Piwik::printMemoryUsage('after archive instanciation');
 $period = new Piwik_Period_Week(Piwik_Date::today());
+Piwik::printMemoryUsage('after period');
 $site = new Piwik_Site(1);
+Piwik::printMemoryUsage('after site');
 $test->setPeriod($period);
 $test->setSite($site);
 $test->get('nb_visits');
+Piwik::printMemoryUsage('after first get');
 $test->get('nb_visits');
 $test->get('nb_visits');
 $test->get('nb_visits');
 $test->get('toto12');
-
+Piwik::log("Referers_keywordBySearchEngine = ". $test->getDataTable('Referers_keywordBySearchEngine'));
 
 //main();
 //displayProfiler();
diff --git a/misc/generateVisits.php b/misc/generateVisits.php
index 883f644aca..89adc70f93 100644
--- a/misc/generateVisits.php
+++ b/misc/generateVisits.php
@@ -33,7 +33,7 @@ ob_start();
 Piwik_PluginsManager::getInstance()->doNotLoadPlugins();	
 $generator = new Piwik_LogStats_Generator;
 $generator->disableProfiler();
-//$generator->emptyAllLogTables();
+$generator->emptyAllLogTables();
 $generator->init();
 
 $t = new Piwik_Timer;
@@ -41,14 +41,14 @@ $t = new Piwik_Timer;
 /*
  * Generate visits / actions for the last 31 days
  */
-$daysToCompute = 25;
+$daysToCompute = 10;
 $startTime = time() - ($daysToCompute-1)*86400;
 $nbActionsTotal = 0;
 while($startTime <= time())
 {
 	$visits = rand(1,2);
 	$actions = 10;
-	$visits = rand(1000,3000);
+	$visits = rand(100,300);
 	$actions = 5;
 	
 	$generator->setTimestampToUse($startTime);
@@ -60,6 +60,8 @@ while($startTime <= time())
 	$nbActionsTotal+=$nbActionsTotalThisDay;
 }
 
+
+echo "<br>Total actions: $nbActionsTotal";
 echo "<br>Total requests per sec: ". round($nbActionsTotal / $t->getTime(),0);
 echo "<br>".$t;
 
diff --git a/modules/Archive.php b/modules/Archive.php
index 522b33e464..d1a52fb558 100644
--- a/modules/Archive.php
+++ b/modules/Archive.php
@@ -101,11 +101,20 @@ class Piwik_Archive
 	
 	public function get( $name, $typeValue = 'numeric' )
 	{
-		if($this->cacheEnabledForNumeric
+		// values previously "get" and now cached
+		if($typeValue == 'numeric'
+			&& $this->cacheEnabledForNumeric
 			&& isset($this->numericCached[$name])
 			)
 		{
-			return $this->numericCached[$name][$typeValue];
+			return $this->numericCached[$name];
+		}
+		
+		// Values prefetched
+		if($typeValue == 'blob'
+			&& isset($this->blobCached[$name]))
+		{
+			return $this->blobCached[$name];
 		}
 		
 		$this->prepareArchive();
@@ -126,16 +135,12 @@ class Piwik_Archive
 		switch($typeValue)
 		{
 			case 'blob':
-				$tableBlob = $this->archiveProcessing->getTableArchiveBlobName();
-				// select data from the blob table
-				$table = $tableBlob; 
+				$table = $this->archiveProcessing->getTableArchiveBlobName();
 			break;
 
 			case 'numeric':
 			default:
-				$tableNumeric = $this->archiveProcessing->getTableArchiveNumericName();
-				// select data from the numeric table (by default)
-				$table = $tableNumeric;
+				$table = $this->archiveProcessing->getTableArchiveNumericName();
 			break;
 		}
 
@@ -151,10 +156,11 @@ class Piwik_Archive
 		// no result, returns false
 		if($value === false)
 		{
-			if($this->cacheEnabledForNumeric)
+			if($typeValue == 'numeric' 
+				&& $this->cacheEnabledForNumeric)
 			{
 				// we cache the results
-				$this->numericCached[$name][$typeValue] = false;
+				$this->numericCached[$name] = false;
 			}	
 			return $value;
 		}
@@ -165,17 +171,18 @@ class Piwik_Archive
 			$value = gzuncompress($value);
 		}
 		
-		if($this->cacheEnabledForNumeric)
+		if($typeValue == 'numeric' 
+			&& $this->cacheEnabledForNumeric)
 		{
 			// we cache the results
-			$this->numericCached[$name][$typeValue] = $value;
+			$this->numericCached[$name] = $value;
 		}
 		return $value;
 	}
 	
 	public function getDataTable( $name, $idSubTable = null )
 	{
-		if($idSubTable !== null)
+		if(!is_null($idSubTable))
 		{
 			$name .= "_$idSubTable";
 		}
@@ -209,14 +216,6 @@ class Piwik_Archive
 		return $this->get($name, 'blob');		
 	}
 	
-	// fetches many fields at once for performance
-	public function preFetchNumeric( $aName )
-	{
-		// TODO implement prefetch
-		
-		
-	}
-	
 	public function freeBlob( $name )
 	{
 		
@@ -247,14 +246,8 @@ class Piwik_Archive
 		{
 			$value = $row['value'];
 			$name = $row['name'];
-			
-			$value = gzuncompress($value);
-			
-			// we cache the results
-			if($this->cacheEnabledForNumeric)
-			{
-				$this->numericCached[$name]['blob'] = $value;
-			}
+						
+			$this->blobCached[$name] = gzuncompress($value);
 		}
 	}
 }
diff --git a/modules/ArchiveProcessing.php b/modules/ArchiveProcessing.php
index d600b196ff..fa7d122561 100644
--- a/modules/ArchiveProcessing.php
+++ b/modules/ArchiveProcessing.php
@@ -100,11 +100,15 @@ abstract class Piwik_ArchiveProcessing
 		$this->idArchive = $this->isArchived();
 		if(!$this->idArchive)
 		{
+			Piwik::printMemoryUsage('Before loading subperiods');
 			$this->archivesSubperiods = $this->loadSubperiodsArchive();
-			
+			Piwik::printMemoryUsage('After loading subperiods');
 			$this->initCompute();
+			Piwik::printMemoryUsage('After init compute');
 			$this->compute();
+			Piwik::printMemoryUsage('After compute');
 			$this->postCompute();
+			Piwik::printMemoryUsage('After post compute');
 			
 			Piwik::log("New archive computed, id = {$this->idArchive}");
 		}
@@ -166,6 +170,7 @@ abstract class Piwik_ArchiveProcessing
 		
 		// we delete all tables from the table register
 		Piwik_ArchiveProcessing_Record_Manager::getInstance()->deleteAll();
+		
 	} 
 	
 	
diff --git a/modules/ArchiveProcessing/Period.php b/modules/ArchiveProcessing/Period.php
index ceb15d41d2..f29beb0eb6 100644
--- a/modules/ArchiveProcessing/Period.php
+++ b/modules/ArchiveProcessing/Period.php
@@ -13,12 +13,10 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 			$aNames = array($aNames);
 		}
 		
-		// fetch the numeric values and sum them
+		// fetch the numeric values and apply the operation on them
 		$results = array();
 		foreach($this->archives as $archive)
 		{
-			$archive->preFetchNumeric($aNames);
-			
 			foreach($aNames as $name)
 			{
 				if(!isset($results[$name]))
@@ -39,7 +37,6 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 						case 'min':
 							$results[$name] = min($results[$name], $valueToSum);		
 							break;
-					
 						default:
 							throw new Exception("Operation not applicable.");
 							break;
@@ -67,10 +64,12 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 		// returns the array of records once summed
 		return $records;
 	}
+	
 	public function archiveNumericValuesSum( $aNames )
 	{
 		return $this->archiveNumericValuesGeneral($aNames, 'sum');
 	}
+	
 	public function archiveNumericValuesMax( $aNames )
 	{
 		return $this->archiveNumericValuesGeneral($aNames, 'max');
@@ -94,7 +93,7 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 	}
 	
 	private function reloadSubtables($name, $dataTableToLoad, $archive)
-	{	
+	{
 		// we have to recursively load all the subtables associated to this table's rows
 		// and update the subtableID so that it matches the newly instanciated table 
 		foreach($dataTableToLoad->getRows() as $row)
@@ -125,8 +124,8 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 			
 //			echo $datatableToSum;
 			$table->addDataTable($datatableToSum);
+			$archive->freeBlob($name);
 		}
-		$archive->freeBlob($name);
 //		echo $table;
 		return $table;
 	}
@@ -146,7 +145,10 @@ class Piwik_ArchiveProcessing_Period extends Piwik_ArchiveProcessing
 		);
 		$this->archiveNumericValuesSum($toSum);
 		
-		Piwik_PostEvent('ArchiveProcessing_Period.compute', $this);
+		Piwik_PostEvent('ArchiveProcessing_Period.compute', $this);		
+		
+		//delete all DataTable instanciated
+		Piwik_DataTable_Manager::getInstance()->deleteAll();
+		
 	}
 }
-?>
diff --git a/modules/ArchiveProcessing/Record.php b/modules/ArchiveProcessing/Record.php
index cab2d76c25..aa775025fd 100644
--- a/modules/ArchiveProcessing/Record.php
+++ b/modules/ArchiveProcessing/Record.php
@@ -49,7 +49,10 @@ class Piwik_ArchiveProcessing_Record_Manager
 	
 	public function deleteAll()
 	{
-		unset($this->records);
+		foreach($this->records as $key => $record)
+		{
+			unset($this->records[$key]);
+		}
 		$this->records = array();
 	}
 }
@@ -71,7 +74,6 @@ abstract class Piwik_ArchiveProcessing_Record
 	}
 	public function __destruct()
 	{
-		Piwik_ArchiveProcessing_Record_Manager::getInstance()->unregister($this);
 	}
 }
 
diff --git a/modules/DataTable/Manager.php b/modules/DataTable/Manager.php
index 7225beaa49..cc93c61089 100644
--- a/modules/DataTable/Manager.php
+++ b/modules/DataTable/Manager.php
@@ -42,12 +42,10 @@ class Piwik_DataTable_Manager
 	
 	function deleteAll()
 	{
-		foreach($this->tables as $key => $table)
-		{
-			unset($this->tables[$key]);
-		}
+		Piwik::log("DELETE ALL ".$this->count()." TABLES");
 		$this->tables = array();
 	}
+	
 	function count()
 	{
 		return count($this->tables);
diff --git a/modules/LogStats/Generator.php b/modules/LogStats/Generator.php
index e2ffb7e175..b5d390e860 100644
--- a/modules/LogStats/Generator.php
+++ b/modules/LogStats/Generator.php
@@ -290,7 +290,7 @@ class Piwik_LogStats_Generator
 			{
 				if(rand(0,2)==1)
 				{
-					$this->setCurrentRequest( 'action_name' , $this->getRandomString(3,3));
+					$this->setCurrentRequest( 'action_name' , $this->getRandomString(1,1));
 				}
 			}
 		}
@@ -305,10 +305,10 @@ class Piwik_LogStats_Generator
 	{
 		$url = $host;
 		
-		$deep = mt_rand(0,2);
+		$deep = mt_rand(0,1);
 		for($i=0;$i<$deep;$i++)
 		{
-			$name = $this->getRandomString(1,2,'alnum');
+			$name = $this->getRandomString(1,1,'alnum');
 			
 			$url .= '/'.$name;
 		}
@@ -321,14 +321,14 @@ class Piwik_LogStats_Generator
 		$len = mt_rand($minLength, $maxLength);
 		
 	    // Register the lower case alphabet array
-	    $alpha = array('a', 'b', 'c', 'd', 'e', 'f', 'g');
+	    $alpha = array('a', 'd', 'e', 'f', 'g');
 	
 	    // Register the upper case alphabet array                    
 	    $ALPHA = array('A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
 	                     'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z');
 	       
 	    // Register the numeric array              
-	    $num = array('1', '2', '3', '4', '5', '6', '7', '8', '9', '0');
+	    $num = array('1', '2', '3',  '8', '9', '0');
 	    
 	    // Register the strange array              
 	    $strange = array('/', '?', '!','"','£','$','%','^','&','*','(',')',' ');
diff --git a/modules/Piwik.php b/modules/Piwik.php
index afeab8b518..45c240ec8e 100755
--- a/modules/Piwik.php
+++ b/modules/Piwik.php
@@ -17,7 +17,7 @@ class Piwik
 			'year'	=>4,
 		);
 		
-	static public function log($message, $priority = Zend_Log::NOTICE)
+	static public function log($message = '', $priority = Zend_Log::NOTICE)
 	{
 		Zend_Registry::get('logger_message')->log($message);
 		Zend_Registry::get('logger_message')->log( "<br>" . PHP_EOL);
@@ -43,12 +43,22 @@ class Piwik
 
 	static public function printMemoryUsage( $prefixString = null )
 	{
-		$usage = round(memory_get_usage() / 1024 / 1024, 2);
+		if(function_exists('xdebug_memory_usage'))
+		{
+			$memory = xdebug_memory_usage();
+		}
+		else
+		{
+			$memory = memory_get_usage();
+		}
+		
+		$usage = round( $memory / 1024 / 1024, 2);
 		if(!is_null($prefixString))
 		{
 			Piwik::log($prefixString);
 		}
 		Piwik::log("Memory usage = $usage Mb");
+		Piwik::log();
 	}
 	
 	static public function isNumeric($value)
diff --git a/modules/Plugin.php b/modules/Plugin.php
index f7c1431b3b..caf7527e33 100644
--- a/modules/Plugin.php
+++ b/modules/Plugin.php
@@ -8,9 +8,8 @@ abstract class Piwik_Plugin
 {
 	function __construct()
 	{
-		Piwik::log("start of ".__CLASS__);
-		Piwik::printMemoryUsage(); 
 	}
+	
 	/**
 	 * Returns the plugin details
 	 */
diff --git a/piwik.js b/piwik.js
index f2fe9f2184..5c0ed30215 100644
--- a/piwik.js
+++ b/piwik.js
@@ -78,7 +78,7 @@ function _pk_getUrlLog( _pk_action_name, _pk_site, _pk_pkurl )
 	var _pk_url = document.location.href;
 	var _pk_da = new Date();
 	var _pk_src = _pk_pkurl
-		+'?url='+_pk_escape(_pk_url)
+		+'?url='+_pk_escape(document.location.href)
 		+'&action_name='+_pk_escape(_pk_action_name)
 		+'&idsite='+_pk_site
 		+'&res='+screen.width+'x'+screen.height
diff --git a/plugins/Actions.php b/plugins/Actions.php
index c502774605..b8ddfab933 100644
--- a/plugins/Actions.php
+++ b/plugins/Actions.php
@@ -171,22 +171,27 @@ class Piwik_Plugin_Actions extends Piwik_Plugin
 		
 		Piwik::log("$modified rows for sum time per action");
 
-		$data = Piwik_ArchiveProcessing_Day::generateDataTable($this->actionsTablesByType[Piwik_LogStats_Action::TYPE_ACTION]);
-		$s = $data->getSerialized();
+		Piwik::printMemoryUsage("Middle of ".get_class($this)." "); 
+		
+		$dataTable = Piwik_ArchiveProcessing_Day::generateDataTable($this->actionsTablesByType[Piwik_LogStats_Action::TYPE_ACTION]);
+		$s = $dataTable->getSerialized();
 		$record = new Piwik_ArchiveProcessing_Record_Blob_Array('Actions_actions', $s);
 		Piwik::log(" Action serialized has ".count($s)." elements");
 		
 		
-		$data = Piwik_ArchiveProcessing_Day::generateDataTable($this->actionsTablesByType[Piwik_LogStats_Action::TYPE_DOWNLOAD]);
-		$s = $data->getSerialized();
+		$dataTable = Piwik_ArchiveProcessing_Day::generateDataTable($this->actionsTablesByType[Piwik_LogStats_Action::TYPE_DOWNLOAD]);
+		$s = $dataTable->getSerialized();
 		$record = new Piwik_ArchiveProcessing_Record_Blob_Array('Actions_downloads', $s);
 		Piwik::log(" Download serialized has ".count($s)." elements");
 		
 		
-		$data = Piwik_ArchiveProcessing_Day::generateDataTable($this->actionsTablesByType[Piwik_LogStats_Action::TYPE_OUTLINK]);
-		$s = $data->getSerialized();
+		$dataTable = Piwik_ArchiveProcessing_Day::generateDataTable($this->actionsTablesByType[Piwik_LogStats_Action::TYPE_OUTLINK]);
+		$s = $dataTable->getSerialized();
 		$record = new Piwik_ArchiveProcessing_Record_Blob_Array('Actions_outlink', $s);
 		Piwik::log(" Outlink serialized has ".count($s)." elements");
+		
+		unset($this->actionsTablesByType);
+		Piwik::printMemoryUsage("End of ".get_class($this)." "); 
 	}
 	
 	static public function getActionCategoryFromName($name)
diff --git a/plugins/Provider.php b/plugins/Provider.php
index 36f1f79267..b8f681d213 100644
--- a/plugins/Provider.php
+++ b/plugins/Provider.php
@@ -68,6 +68,7 @@ class Piwik_Plugin_Provider extends Piwik_Plugin
 		$tableProvider = $this->ArchiveProcessing->getDataTableInterestForLabel($labelSQL);
 		$record = new Piwik_ArchiveProcessing_Record_Blob_Array($recordName, $tableProvider->getSerialized());
 //		echo $tableProvider;
+		Piwik::printMemoryUsage("End of ".get_class($this)." "); 
 	}
 	
 	/**
diff --git a/plugins/Referers.php b/plugins/Referers.php
index 25b0af6054..8568e2fa08 100644
--- a/plugins/Referers.php
+++ b/plugins/Referers.php
@@ -199,6 +199,7 @@ class Piwik_Plugin_Referers extends Piwik_Plugin
 //		Piwik::log("Urls by partner website:");
 //		Piwik::log($urlByWebsite[Piwik_Common::REFERER_TYPE_PARTNER]);
 		
+		Piwik::printMemoryUsage("Middle of ".get_class($this)." "); 
 
 		$data = $archiveProcessing->getDataTableSerialized($interestByType);
 		$record = new Piwik_ArchiveProcessing_Record_Blob_Array('Referers_type', $data);
@@ -220,6 +221,7 @@ class Piwik_Plugin_Referers extends Piwik_Plugin
 		$data = $archiveProcessing->getDataTablesSerialized($urlByWebsite[Piwik_Common::REFERER_TYPE_PARTNER], $interestByWebsite[Piwik_Common::REFERER_TYPE_PARTNER]);
 		$record = new Piwik_ArchiveProcessing_Record_Blob_Array('Referers_urlByPartner', $data);
 			
+		Piwik::printMemoryUsage("End of ".get_class($this)." "); 
 //		echo "after serialization = ". $timer;
 	}
 }
\ No newline at end of file
diff --git a/plugins/UserCountry.php b/plugins/UserCountry.php
index ccab34c368..6f8bfc89bb 100644
--- a/plugins/UserCountry.php
+++ b/plugins/UserCountry.php
@@ -66,5 +66,6 @@ class Piwik_Plugin_UserCountry extends Piwik_Plugin
 		$tableContinent = $this->ArchiveProcessing->getDataTableInterestForLabel($labelSQL);
 		$record = new Piwik_ArchiveProcessing_Record_Blob_Array($recordName, $tableContinent->getSerialized());
 //		echo $tableContinent;
+		Piwik::printMemoryUsage("End of ".get_class($this)." "); 
 	}
 }
\ No newline at end of file
diff --git a/plugins/UserSettings.php b/plugins/UserSettings.php
index 005b89390a..2a59268e25 100644
--- a/plugins/UserSettings.php
+++ b/plugins/UserSettings.php
@@ -164,6 +164,7 @@ class Piwik_Plugin_UserSettings extends Piwik_Plugin
 //		echo $tableResolution;
 //		echo $tableWideScreen;
 //		echo $tablePlugin;
+		Piwik::printMemoryUsage("End of ".get_class($this)." "); 
 	}
 	
 	function archiveMonth( $notification )