1 files changed, 186 insertions, 70 deletions

diff --git a/core/ArchiveProcessing.php b/core/ArchiveProcessing.php
index 91eed568a4..23dca365cf 100644
--- a/core/ArchiveProcessing.php
+++ b/core/ArchiveProcessing.php
@@ -45,6 +45,14 @@ abstract class Piwik_ArchiveProcessing
 	 * @var int
 	 */
 	const DONE_ERROR = 2;
+	
+	/**
+	 * Flag indicates the archive is over a period that is not finished, eg. the current day, current week, etc.
+	 * Archives flagged will be regularly purged from the DB.
+	 * 
+	 * @var int
+	 */
+	const DONE_OK_TEMPORARY = 3;
 
 	/**
 	 * Idarchive in the DB for the requested archive
@@ -96,11 +104,11 @@ abstract class Piwik_ArchiveProcessing
 	protected $tableArchiveBlob;
 	
 	/**
-	 * Maximum timestamp above which a given archive is considered out of date 
+	 * Minimum timestamp looked at for processed archives  
 	 *
 	 * @var int
 	 */
-	protected $maxTimestampArchive;
+	protected $minDatetimeArchiveProcessedUTC = false;
 
 	/**
 	 * Compress blobs
@@ -110,6 +118,13 @@ abstract class Piwik_ArchiveProcessing
 	protected $compressBlob;
 	
 	/**
+	 * Is the current archive temporary. ie.
+	 * - today 
+	 * - current week / month / year
+	 */
+	protected $temporaryArchive;
+	
+	/**
 	 * Id of the current site
 	 * Can be accessed by plugins (that is why it's public)
 	 * 
@@ -121,7 +136,7 @@ abstract class Piwik_ArchiveProcessing
 	 * Period of the current archive
 	 * Can be accessed by plugins (that is why it's public)
 	 * 
-	 * @var Piwik_Period
+	 * @var $period Piwik_Period
 	 */
 	public $period 	= null;
 	
@@ -134,14 +149,14 @@ abstract class Piwik_ArchiveProcessing
 	public $site 	= null;
 	
 	/**
-	 * Starting date @see Piwik_Date::toString()
+	 * Starting datetime in UTC
 	 *
 	 * @var string
 	 */
-	public $strDateStart;
+	public $startDatetimeUTC;
 	
 	/**
-	 * Ending date @see Piwik_Date::toString()
+	 * Ending date in UTC
 	 *
 	 * @var string
 	 */
@@ -183,6 +198,9 @@ abstract class Piwik_ArchiveProcessing
 	 */
 	public $isThereSomeVisits = false;
 	
+	protected $startTimestampUTC;
+	protected $endTimestampUTC;
+	
 	/**
 	 * Constructor
 	 */
@@ -218,61 +236,142 @@ abstract class Piwik_ArchiveProcessing
 		return $process;
 	}
 	
+	const OPTION_TODAY_ARCHIVE_TTL = 'todayArchiveTimeToLive';
+	const OPTION_BROWSER_TRIGGER_ARCHIVING = 'enableBrowserTriggerArchiving';
+	
+	static public function setTodayArchiveTimeToLive($timeToLiveSeconds)
+	{
+		$timeToLiveSeconds = (int)$timeToLiveSeconds;
+		if($timeToLiveSeconds <= 0)
+		{
+			throw new Exception(Piwik_TranslateException('General_ExceptionInvalidArchiveTimeToLive'));
+		}
+		Piwik_SetOption(self::OPTION_TODAY_ARCHIVE_TTL, $timeToLiveSeconds, $autoload = true);
+	}
+	
+	static public function getTodayArchiveTimeToLive()
+	{
+		$timeToLive = Piwik_GetOption(self::OPTION_TODAY_ARCHIVE_TTL);
+		if($timeToLive !== false)
+		{
+			return $timeToLive;
+		}
+		return Zend_Registry::get('config')->General->time_before_today_archive_considered_outdated;
+	}
+
+	static public function setBrowserTriggerArchiving($enabled)
+	{
+		if(!is_bool($enabled))
+		{
+			throw new Exception('Browser trigger archiving must be set to true or false.');
+		}
+		Piwik_SetOption(self::OPTION_BROWSER_TRIGGER_ARCHIVING, (int)$enabled, $autoload = true);
+		
+	}
+	static public function isBrowserTriggerArchivingEnabled()
+	{
+		$browserArchivingEnabled = Piwik_GetOption(self::OPTION_BROWSER_TRIGGER_ARCHIVING);
+		if($browserArchivingEnabled !== false)
+		{
+			return (bool)$browserArchivingEnabled;
+		}
+		return (bool)Zend_Registry::get('config')->General->enable_browser_archiving_triggering;
+	}
+	
 	public function getIdArchive()
 	{
 		return $this->idArchive;
 	}
 	
 	/**
-	 * Inits the object
+	 * Sets object attributes that will be used throughout the process
 	 */
-	protected function loadArchiveProperties()
-	{		
+	public function init()
+	{
 		$this->idsite = $this->site->getId();
-		
 		$this->periodId = $this->period->getId();
-		
-		$this->dateStart = $this->period->getDateStart();
-		$this->dateEnd = $this->period->getDateEnd();
+
+		$dateStartLocalTimezone = $this->period->getDateStart();
+		$dateEndLocalTimezone = $this->period->getDateEnd();
 		
 		$this->tableArchiveNumeric = new Piwik_TablePartitioning_Monthly('archive_numeric');
 		$this->tableArchiveNumeric->setIdSite($this->idsite);
-		$this->tableArchiveNumeric->setTimestamp($this->dateStart->get());
+		$this->tableArchiveNumeric->setTimestamp($dateStartLocalTimezone->getTimestamp());
 		$this->tableArchiveBlob = new Piwik_TablePartitioning_Monthly('archive_blob');
 		$this->tableArchiveBlob->setIdSite($this->idsite);	
-		$this->tableArchiveBlob->setTimestamp($this->dateStart->get());
+		$this->tableArchiveBlob->setTimestamp($dateStartLocalTimezone->getTimestamp());
+
+		$dateStartUTC = $dateStartLocalTimezone->setTimezone($this->site->getTimezone());
+		$dateEndUTC = $dateEndLocalTimezone->setTimezone($this->site->getTimezone());
+		$this->startDatetimeUTC = $dateStartUTC->getDateStartUTC();
+		$this->endDatetimeUTC = $dateEndUTC->getDateEndUTC();
 
-		$this->strDateStart = $this->dateStart->toString();
-		$this->strDateEnd = $this->dateEnd->toString();
+		$this->startTimestampUTC = $dateStartUTC->getTimestamp();
+		$this->endTimestampUTC = strtotime($this->endDatetimeUTC);
 		
+		$this->minDatetimeArchiveProcessedUTC = $this->getMinTimeArchivedProcessed();
+		$db = Zend_Registry::get('db');
+		$this->compressBlob = $db->hasBlobDataType();
+	}
+
+	public function getStartDatetimeUTC()
+	{
+		return $this->startDatetimeUTC;
+	}
+	
+	public function getEndDatetimeUTC()
+	{
+		return $this->endDatetimeUTC;
+	}
+	
+	public function isArchiveTemporary()
+	{
+		return $this->temporaryArchive;
+	}
+	
+	/**
+	 * Returns the minimum archive processed datetime to look at
+	 *  
+	 * @return string Datetime string, or false if must look at any archive available
+	 */
+	public function getMinTimeArchivedProcessed()
+	{
+		$this->temporaryArchive = false;
 		// if the current archive is a DAY and if it's today,
-		// we set this maxTimestampArchive that defines the lifetime value of today's archive
-		$this->maxTimestampArchive = 0;
+		// we set this minDatetimeArchiveProcessedUTC that defines the lifetime value of today's archive
 		if( $this->period->getNumberOfSubperiods() == 0
-			&& $this->period->toString() == date("Y-m-d")
+			&& $this->startTimestampUTC <= time() && $this->endTimestampUTC > time()
 			)
 		{
-			$this->maxTimestampArchive = time() - Zend_Registry::get('config')->General->time_before_today_archive_considered_outdated;
+			$this->temporaryArchive = true;
+			$minDatetimeArchiveProcessedUTC = time() - self::getTodayArchiveTimeToLive();
+			// see #1150; if new archives are not triggered from the browser, 
+			// we still want to try and return the latest archive available for today (rather than return nothing)
+			if($this->isArchivingDisabled())
+			{
+				return false;
+			}
 		}
 		// either
 		// - if the period we're looking for is finished, we look for a ts_archived that 
 		//   is greater than the last day of the archive 
 		// - if the period we're looking for is not finished, we look for a recent enough archive
-		//   recent enough means maxTimestampArchive = 00:00:01 this morning
+		//   recent enough means minDatetimeArchiveProcessedUTC = 00:00:01 this morning
 		else
 		{
-			if($this->period->isFinished())
+			if($this->endTimestampUTC <= time())
 			{
-				$this->maxTimestampArchive = $this->period->getDateEnd()->setTime('00:00:00')->addDay(1)->getTimestamp();
+				$minDatetimeArchiveProcessedUTC = $this->endTimestampUTC+1;
 			}
 			else
 			{
-				$this->maxTimestampArchive = Piwik_Date::today()->getTimestamp();
+    			$this->temporaryArchive = true;
+				$minDatetimeArchiveProcessedUTC = Piwik_Date::today()
+													->setTimezone($this->site->getTimezone())
+													->getTimestamp();
 			}
 		}
-
-		$db = Zend_Registry::get('db');
-		$this->compressBlob = $db->hasBlobDataType();
+		return $minDatetimeArchiveProcessedUTC;
 	}
 	
 	/**
@@ -286,7 +385,7 @@ abstract class Piwik_ArchiveProcessing
 	 */
 	public function loadArchive()
 	{
-		$this->loadArchiveProperties();
+		$this->init();
 		$this->idArchive = $this->isArchived();
 	
 		if($this->idArchive === false
@@ -327,10 +426,19 @@ abstract class Piwik_ArchiveProcessing
 	{
 		$this->loadNextIdarchive();
 		$this->insertNumericRecord('done', Piwik_ArchiveProcessing::DONE_ERROR);
-		$this->logTable 			= Piwik::prefixTable('log_visit');
-		$this->logVisitActionTable 	= Piwik::prefixTable('log_link_visit_action');
-		$this->logActionTable	 	= Piwik::prefixTable('log_action');
-		$this->logConversionTable	= Piwik::prefixTable('log_conversion');
+		$this->logTable 			= Piwik_Common::prefixTable('log_visit');
+		$this->logVisitActionTable 	= Piwik_Common::prefixTable('log_link_visit_action');
+		$this->logActionTable	 	= Piwik_Common::prefixTable('log_action');
+		$this->logConversionTable	= Piwik_Common::prefixTable('log_conversion');
+		
+		$temporary = 'definitive archive';
+		if($this->isArchiveTemporary())
+		{
+			$temporary = 'temporary archive';
+		}
+		Piwik::log("Processing archive '" . $this->period->getLabel() . "', 
+								idsite = ". $this->idsite." ($temporary) - 
+								UTC datetime [".$this->startDatetimeUTC." -> ".$this->endDatetimeUTC." ]...");
 	}
 	
 	/**
@@ -343,12 +451,17 @@ abstract class Piwik_ArchiveProcessing
 	{
 		// delete the first done = ERROR 
 		Piwik_Query("/* SHARDING_ID_SITE = ".$this->idsite." */ 
-							DELETE FROM ".$this->tableArchiveNumeric->getTableName()." 
-							WHERE idarchive = ? AND name = 'done'",
+					DELETE FROM ".$this->tableArchiveNumeric->getTableName()." 
+					WHERE idarchive = ? AND name = 'done'",
 					array($this->idArchive)
-				);
+		);
 		
-		$this->insertNumericRecord('done', Piwik_ArchiveProcessing::DONE_OK);
+		$flag = Piwik_ArchiveProcessing::DONE_OK;
+		if($this->isArchiveTemporary())
+		{
+			$flag = Piwik_ArchiveProcessing::DONE_OK_TEMPORARY;
+		}
+		$this->insertNumericRecord('done', $flag);
 		
 		Piwik_DataTable_Manager::getInstance()->deleteAll();
 	}
@@ -400,14 +513,8 @@ abstract class Piwik_ArchiveProcessing
 	 */
 	public function getTimestampStartDate()
 	{
-		// case when archive processing is in the past or the future, the starting date has not been set or processed yet
-		if(is_null($this->timestampDateStart))
-		{
-			return Piwik_Date::factory($this->strDateStart)->getTimestamp();
-		}
 		return $this->timestampDateStart;
 	}
-	
 
 	// exposing the number of visits publicly (number used to compute conversions rates)
 	protected $nb_visits = null;
@@ -438,7 +545,9 @@ abstract class Piwik_ArchiveProcessing
 	protected function loadNextIdarchive()
 	{
 		$db = Zend_Registry::get('db');
-		$id = $db->fetchOne("/* SHARDING_ID_SITE = ".$this->idsite." */ SELECT max(idarchive) FROM ".$this->tableArchiveNumeric->getTableName());
+		$id = $db->fetchOne("/* SHARDING_ID_SITE = ".$this->idsite." */ 
+						SELECT max(idarchive) 
+						FROM ".$this->tableArchiveNumeric->getTableName());
 		if(empty($id))
 		{
 			$id = 0;
@@ -498,23 +607,31 @@ abstract class Piwik_ArchiveProcessing
 	 */
 	protected function insertRecord($record)
 	{
+		
 		// table to use to save the data
 		if(is_numeric($record->value))
 		{
+    		// We choose not to record records with a value of 0 
+    		if($record->value == 0)
+    		{
+    			return;
+    		}
 			$table = $this->tableArchiveNumeric;
 		}
 		else
 		{
 			$table = $this->tableArchiveBlob;
 		}
-		
-		$query = "INSERT INTO ".$table->getTableName()." (idarchive, idsite, date1, date2, period, ts_archived, name, value)
+
+		// ignore duplicate idarchive
+		// @see http://dev.piwik.org/trac/ticket/987
+		$query = "INSERT IGNORE INTO ".$table->getTableName()." (idarchive, idsite, date1, date2, period, ts_archived, name, value)
 					VALUES (?,?,?,?,?,?,?,?)";
 		Piwik_Query($query, 
 							array(	$this->idArchive,
 									$this->idsite, 
-									$this->strDateStart, 
-									$this->strDateEnd, 
+									$this->period->getDateStart()->toString('Y-m-d'), 
+									$this->period->getDateEnd()->toString('Y-m-d'), 
 									$this->periodId, 
 									date("Y-m-d H:i:s"),
 									$record->name,
@@ -528,7 +645,7 @@ abstract class Piwik_ArchiveProcessing
 	 * Returns false if the archive needs to be computed.
 	 * 
 	 * An archive is available if
-	 * - for today, the archive was computed less than maxTimestampArchive seconds ago
+	 * - for today, the archive was computed less than minDatetimeArchiveProcessedUTC seconds ago
 	 * - for any other day, if the archive was computed once this day was finished
 	 * - for other periods, if the archive was computed once the period was finished
 	 *
@@ -537,20 +654,27 @@ abstract class Piwik_ArchiveProcessing
 	protected function isArchived()
 	{
 		$bindSQL = array(	$this->idsite, 
-								$this->strDateStart, 
-								$this->strDateEnd, 
-								$this->periodId, 
-								);
-		$timeStampWhere = " AND UNIX_TIMESTAMP(ts_archived) >= ? ";
-		$bindSQL[] = $this->maxTimestampArchive;
+							$this->period->getDateStart()->toString('Y-m-d'), 
+							$this->period->getDateEnd()->toString('Y-m-d'), 
+							$this->periodId, 
+		);
+		
+		$timeStampWhere = '';
 		
-		$sqlQuery = "	SELECT idarchive, value, name, UNIX_TIMESTAMP(date1) as timestamp
+		if($this->minDatetimeArchiveProcessedUTC)
+		{
+    		$timeStampWhere = " AND ts_archived >= ? ";
+    		$bindSQL[] = Piwik_Date::factory($this->minDatetimeArchiveProcessedUTC)->getDatetime();
+		}
+		
+		$sqlQuery = "	SELECT idarchive, value, name, date1 as startDate
 						FROM ".$this->tableArchiveNumeric->getTableName()."
 						WHERE idsite = ?
 							AND date1 = ?
 							AND date2 = ?
 							AND period = ?
 							AND ( (name = 'done' AND value = ".Piwik_ArchiveProcessing::DONE_OK.")
+									OR (name = 'done' AND value = ".Piwik_ArchiveProcessing::DONE_OK_TEMPORARY.")
 									OR name = 'nb_visits')
 							$timeStampWhere
 						ORDER BY ts_archived DESC";
@@ -567,7 +691,7 @@ abstract class Piwik_ArchiveProcessing
 			if($result['name'] == 'done')
 			{
 				$idarchive = $result['idarchive'];
-				$this->timestampDateStart = $result['timestamp'];
+				$this->timestampDateStart = Piwik_Date::factory($result['startDate'])->getTimestamp();
 				break;
 			}
 		}
@@ -579,7 +703,7 @@ abstract class Piwik_ArchiveProcessing
 			return false;
 		}
 		
-		// we look for the nb_visits result for this more recent archive
+		// we look for the nb_visits result for this most recent archive
 		foreach($results as $result)
 		{
 			if($result['name'] == 'nb_visits' 
@@ -599,19 +723,11 @@ abstract class Piwik_ArchiveProcessing
 	 */
 	protected function isArchivingDisabled()
 	{
-		static $archivingIsDisabled = null;
-		if(is_null($archivingIsDisabled))
+		if(!self::isBrowserTriggerArchivingEnabled()
+			&& !Piwik_Common::isPhpCliMode())
 		{
-			$archivingIsDisabled = false;
-			$enableBrowserArchivingTriggering = (bool)Zend_Registry::get('config')->General->enable_browser_archiving_triggering;
-			if($enableBrowserArchivingTriggering == false)
-			{
-				if( !Piwik_Common::isPhpCliMode())
-				{
-					$archivingIsDisabled = true;
-				}
-			}
+			return true;
 		}
-		return $archivingIsDisabled;
+		return false;
 	}
 }