diff options
-rw-r--r-- | core/Archive/ArchivePurger.php | 58 | ||||
-rw-r--r-- | core/CronArchive.php | 11 | ||||
-rw-r--r-- | core/DataAccess/Model.php | 35 | ||||
-rw-r--r-- | core/Scheduler/Scheduler.php | 18 | ||||
-rw-r--r-- | core/Scheduler/Timetable.php | 6 | ||||
m--------- | misc/log-analytics | 0 | ||||
-rw-r--r-- | plugins/CoreAdminHome/API.php | 16 | ||||
-rw-r--r-- | plugins/CoreAdminHome/Tasks.php | 41 | ||||
m--------- | plugins/QueuedTracking | 0 | ||||
m--------- | plugins/SecurityInfo | 0 | ||||
m--------- | plugins/TasksTimetable | 0 | ||||
m--------- | plugins/VisitorGenerator | 0 |
12 files changed, 137 insertions, 48 deletions
diff --git a/core/Archive/ArchivePurger.php b/core/Archive/ArchivePurger.php index e41ce375c5..910a539dbb 100644 --- a/core/Archive/ArchivePurger.php +++ b/core/Archive/ArchivePurger.php @@ -10,12 +10,13 @@ namespace Piwik\Archive; use Piwik\ArchiveProcessor\Rules; use Piwik\Config; +use Piwik\Container\StaticContainer; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\DataAccess\Model; use Piwik\Date; use Piwik\Db; -use Piwik\Log; use Piwik\Piwik; +use Psr\Log\LoggerInterface; /** * Service that purges temporary, error-ed, invalid and custom range archives from archive tables. @@ -64,7 +65,12 @@ class ArchivePurger */ private $now; - public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null) + /** + * @var LoggerInterface + */ + private $logger; + + public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null, LoggerInterface $logger = null) { $this->model = $model ?: new Model(); @@ -73,6 +79,7 @@ class ArchivePurger $this->yesterday = Date::factory('yesterday'); $this->today = Date::factory('today'); $this->now = time(); + $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface'); } /** @@ -90,15 +97,25 @@ class ArchivePurger // the constraint will hit an INDEX and speed up the inner join that happens in getInvalidatedArchiveIdsSafeToDelete(). $idSites = $this->model->getSitesWithInvalidatedArchive($numericTable); if (empty($idSites)) { + $this->logger->info("No sites with invalidated archives found in {table}.", array('table' => $numericTable)); return; } $archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites); if (empty($archiveIds)) { + $this->logger->info("No invalidated archives found in {table} with newer, valid archives.", array('table' => $numericTable)); return; } - $this->deleteArchiveIds($date, $archiveIds); + $this->logger->info("Found {countArchiveIds} invalidated archives safe to delete in {table}.", array( + 'table' => $numericTable, 'countArchiveIds' => count($archiveIds) + )); + + $deletedRowCount = $this->deleteArchiveIds($date, $archiveIds); + + $this->logger->info("Deleted {count} rows in {table} and its associated blob table.", array( + 'table' => $numericTable, 'count' => $deletedRowCount + )); } /** @@ -113,13 +130,21 @@ class ArchivePurger $idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan); if (!empty($idArchivesToDelete)) { - $this->deleteArchiveIds($dateStart, $idArchivesToDelete); + $deletedRowCount = $this->deleteArchiveIds($dateStart, $idArchivesToDelete); + + $this->logger->info("Deleted {count} rows in archive tables (numeric + blob) for {date}.", array( + 'count' => $deletedRowCount, + 'date' => $dateStart + )); + } else { + $this->logger->info("No outdated archives found in archive numeric table for {date}.", array('date' => $dateStart)); } - Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]", - $purgeArchivesOlderThan, - $dateStart->toString("Y-m"), - implode(',', $idArchivesToDelete)); + $this->logger->debug("Purging temporary archives: done [ purged archives older than {date} in {yearMonth} ] [Deleted IDs: {deletedIds}]", array( + 'date' => $purgeArchivesOlderThan, + 'yearMonth' => $dateStart->toString('Y-m'), + 'deletedIds' => implode(',', $idArchivesToDelete) + )); } protected function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan) @@ -148,10 +173,16 @@ class ArchivePurger $numericTable = ArchiveTableCreator::getNumericTable($date); $blobTable = ArchiveTableCreator::getBlobTable($date); - $this->model->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan); + $deletedCount = $this->model->deleteArchivesWithPeriod( + $numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan); + + $this->logger->info("Purged {count} range archive rows from {numericTable} & {blobTable}.", array( + 'count' => $deletedCount, + 'numericTable' => $numericTable, + 'blobTable' => $blobTable + )); - Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]", - $this->purgeCustomRangesOlderThan, $numericTable); + $this->logger->debug(" [ purged archives older than {threshold} ]", array('threshold' => $this->purgeCustomRangesOlderThan)); } /** @@ -159,6 +190,7 @@ class ArchivePurger * * @param Date $date * @param $idArchivesToDelete + * @return int Number of rows deleted from both numeric + blob table. */ protected function deleteArchiveIds(Date $date, $idArchivesToDelete) { @@ -166,9 +198,11 @@ class ArchivePurger $numericTable = ArchiveTableCreator::getNumericTable($date); $blobTable = ArchiveTableCreator::getBlobTable($date); + $deletedCount = 0; foreach ($batches as $idsToDelete) { - $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete); + $deletedCount += $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete); } + return $deletedCount; } /** diff --git a/core/CronArchive.php b/core/CronArchive.php index 50716ea756..82b7e2922b 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -429,13 +429,12 @@ class CronArchive $this->log("Starting Scheduled tasks... "); - $tasksOutput = $this->request("?module=API&method=CoreAdminHome.runScheduledTasks&format=csv&convertToUnicode=0&token_auth=" . $this->token_auth); + API\Request::processRequest("CoreAdminHome.runScheduledTasks", array( + 'token_auth' => $this->token_auth, + 'format' => 'original', // so exceptions get thrown + 'trigger' => 'archivephp' + )); - if ($tasksOutput == \Piwik\DataTable\Renderer\Csv::NO_DATA_AVAILABLE) { - $tasksOutput = " No task to run"; - } - - $this->log($tasksOutput); $this->log("done"); $this->logSection(""); } diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php index 0980983fe3..8df49d16a3 100644 --- a/core/DataAccess/Model.php +++ b/core/DataAccess/Model.php @@ -10,9 +10,11 @@ namespace Piwik\DataAccess; use Exception; use Piwik\Common; +use Piwik\Container\StaticContainer; use Piwik\Db; use Piwik\DbHelper; use Piwik\Sequence; +use Psr\Log\LoggerInterface; /** * Cleans up outdated archives @@ -21,6 +23,15 @@ use Piwik\Sequence; */ class Model { + /** + * @var LoggerInterface + */ + private $logger; + + public function __construct(LoggerInterface $logger = null) + { + $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface'); + } /** * Returns the archives IDs that have already been invalidated and have been since re-processed. @@ -118,13 +129,21 @@ class Model $query = "DELETE FROM %s WHERE period = ? AND ts_archived < ?"; $bind = array($period, $date); - Db::query(sprintf($query, $numericTable), $bind); + $queryObj = Db::query(sprintf($query, $numericTable), $bind); + $deletedRows = $queryObj->rowCount(); try { - Db::query(sprintf($query, $blobTable), $bind); + $queryObj = Db::query(sprintf($query, $blobTable), $bind); + $deletedRows += $queryObj->rowCount(); } catch (Exception $e) { // Individual blob tables could be missing + $this->logger->debug("Unable to delete archives by period from {blobTable}.", array( + 'blobTable' => $blobTable, + 'exception' => $e + )); } + + return $deletedRows; } public function deleteArchiveIds($numericTable, $blobTable, $idsToDelete) @@ -132,13 +151,21 @@ class Model $idsToDelete = array_values($idsToDelete); $query = "DELETE FROM %s WHERE idarchive IN (" . Common::getSqlStringFieldsArray($idsToDelete) . ")"; - Db::query(sprintf($query, $numericTable), $idsToDelete); + $queryObj = Db::query(sprintf($query, $numericTable), $idsToDelete); + $deletedRows = $queryObj->rowCount(); try { - Db::query(sprintf($query, $blobTable), $idsToDelete); + $queryObj = Db::query(sprintf($query, $blobTable), $idsToDelete); + $deletedRows += $queryObj->rowCount(); } catch (Exception $e) { // Individual blob tables could be missing + $this->logger->debug("Unable to delete archive IDs from {blobTable}.", array( + 'blobTable' => $blobTable, + 'exception' => $e + )); } + + return $deletedRows; } public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues) diff --git a/core/Scheduler/Scheduler.php b/core/Scheduler/Scheduler.php index 7156e1be9c..f4e2097ca5 100644 --- a/core/Scheduler/Scheduler.php +++ b/core/Scheduler/Scheduler.php @@ -99,9 +99,9 @@ class Scheduler // for every priority level, starting with the highest and concluding with the lowest $executionResults = array(); - for ($priority = Task::HIGHEST_PRIORITY; - $priority <= Task::LOWEST_PRIORITY; - ++$priority) { + for ($priority = Task::HIGHEST_PRIORITY; $priority <= Task::LOWEST_PRIORITY; ++$priority) { + $this->logger->debug("Executing tasks with priority {priority}:", array('priority' => $priority)); + // loop through each task foreach ($tasks as $task) { // if the task does not have the current priority level, don't execute it yet @@ -113,14 +113,24 @@ class Scheduler $shouldExecuteTask = $this->timetable->shouldExecuteTask($taskName); if ($this->timetable->taskShouldBeRescheduled($taskName)) { - $this->timetable->rescheduleTask($task); + $rescheduledDate = $this->timetable->rescheduleTask($task); + + $this->logger->debug("Task {task} is scheduled to run again for {date}.", array('task' => $taskName, 'date' => $rescheduledDate)); } if ($shouldExecuteTask) { + $this->logger->info("Scheduler: executing task {taskName}...", array('taskName' => $taskName)); + + $timer = new Timer(); + $this->isRunningTask = true; $message = $this->executeTask($task); $this->isRunningTask = false; + $this->logger->info("Scheduler: finished executing task {taskName}. {timeElapsed}", array( + 'taskName' => $taskName, 'timeElapsed' => $timer + )); + $executionResults[] = array('task' => $taskName, 'output' => $message); } } diff --git a/core/Scheduler/Timetable.php b/core/Scheduler/Timetable.php index 5c9bc69696..c3c01ab167 100644 --- a/core/Scheduler/Timetable.php +++ b/core/Scheduler/Timetable.php @@ -105,9 +105,13 @@ class Timetable public function rescheduleTask(Task $task) { + $rescheduledTime = $task->getRescheduledTime(); + // update the scheduled time - $this->timetable[$task->getName()] = $task->getRescheduledTime(); + $this->timetable[$task->getName()] = $rescheduledTime; $this->save(); + + return Date::factory($rescheduledTime); } public function save() diff --git a/misc/log-analytics b/misc/log-analytics -Subproject 05dba484f1933a710f171de9b169ee03d240710 +Subproject a4d1eea78e6ea52b1b72e63a494f02e9d6f7223 diff --git a/plugins/CoreAdminHome/API.php b/plugins/CoreAdminHome/API.php index 6329fc8993..2b809b792a 100644 --- a/plugins/CoreAdminHome/API.php +++ b/plugins/CoreAdminHome/API.php @@ -15,6 +15,7 @@ use Piwik\Db; use Piwik\Piwik; use Piwik\Scheduler\Scheduler; use Piwik\Site; +use Psr\Log\LoggerInterface; /** * @method static \Piwik\Plugins\CoreAdminHome\API getInstance() @@ -22,6 +23,16 @@ use Piwik\Site; class API extends \Piwik\Plugin\API { /** + * @var Scheduler + */ + private $scheduler; + + public function __construct(Scheduler $scheduler) + { + $this->scheduler = $scheduler; + } + + /** * Will run all scheduled tasks due to run at this time. * * @return array @@ -31,10 +42,7 @@ class API extends \Piwik\Plugin\API { Piwik::checkUserHasSuperUserAccess(); - /** @var Scheduler $scheduler */ - $scheduler = StaticContainer::getContainer()->get('Piwik\Scheduler\Scheduler'); - - return $scheduler->run(); + return $this->scheduler->run(); } /** diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php index 700c7d69c2..7a3ff406b5 100644 --- a/plugins/CoreAdminHome/Tasks.php +++ b/plugins/CoreAdminHome/Tasks.php @@ -14,9 +14,8 @@ use Piwik\Container\StaticContainer; use Piwik\DataAccess\ArchiveTableCreator; use Piwik\Date; use Piwik\Db; -use Piwik\Log; use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList; -use Piwik\SettingsServer; +use Psr\Log\LoggerInterface; class Tasks extends \Piwik\Plugin\Tasks { @@ -25,9 +24,15 @@ class Tasks extends \Piwik\Plugin\Tasks */ private $archivePurger; - public function __construct(ArchivePurger $archivePurger = null) + /** + * @var LoggerInterface + */ + private $logger; + + public function __construct(ArchivePurger $archivePurger, LoggerInterface $logger) { - $this->archivePurger = $archivePurger ?: new ArchivePurger(); + $this->archivePurger = $archivePurger; + $this->logger = $logger; } public function schedule() @@ -44,16 +49,14 @@ class Tasks extends \Piwik\Plugin\Tasks public function purgeOutdatedArchives() { - $logger = StaticContainer::get('Psr\Log\LoggerInterface'); - if ($this->willPurgingCausePotentialProblemInUI()) { - $logger->info("Purging temporary archives: skipped (browser triggered archiving not enabled & not running after core:archive)"); + $this->logger->info("Purging temporary archives: skipped (browser triggered archiving not enabled & not running after core:archive)"); return false; } $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled(); - $logger->info("Purging archives in {tableCount} archive tables.", array('tableCount' => count($archiveTables))); + $this->logger->info("Purging archives in {tableCount} archive tables.", array('tableCount' => count($archiveTables))); // keep track of dates we purge for, since getTablesArchivesInstalled() will return numeric & blob // tables (so dates will appear two times, and we should only purge once per date) @@ -64,15 +67,19 @@ class Tasks extends \Piwik\Plugin\Tasks list($year, $month) = explode('_', $date); // Somehow we may have archive tables created with older dates, prevent exception from being thrown - if ($year > 1990 - && empty($datesPurged[$date]) - ) { - $dateObj = Date::factory("$year-$month-15"); - - $this->archivePurger->purgeOutdatedArchives($dateObj); - $this->archivePurger->purgeArchivesWithPeriodRange($dateObj); - - $datesPurged[$date] = true; + if ($year > 1990) { + if (empty($datesPurged[$date])) { + $dateObj = Date::factory("$year-$month-15"); + + $this->archivePurger->purgeOutdatedArchives($dateObj); + $this->archivePurger->purgeArchivesWithPeriodRange($dateObj); + + $datesPurged[$date] = true; + } else { + $this->logger->debug("Date {date} already purged.", array('date' => $date)); + } + } else { + $this->logger->info("Skipping purging of archive tables *_{year}_{month}, year <= 1990.", array('year' => $year, 'month' => $month)); } } } diff --git a/plugins/QueuedTracking b/plugins/QueuedTracking -Subproject bc39d75e4c9df58c865ae44f6eae23676e7baf9 +Subproject f1320c0d103942a4a07e6b724a32c17f0b8c6cf diff --git a/plugins/SecurityInfo b/plugins/SecurityInfo -Subproject 85a04e6f1a4b6da0ea821ba2827dd093bbc1277 +Subproject 101aa27943576a3d666ddad24710af2e4c46306 diff --git a/plugins/TasksTimetable b/plugins/TasksTimetable -Subproject a0f38fec7cf905d34059ec1b6a207156adf7579 +Subproject 15197bb051c3c2281d34365d96cc761e621cef9 diff --git a/plugins/VisitorGenerator b/plugins/VisitorGenerator -Subproject 34d796330e256e947ff976f57e350c775dc061d +Subproject a1626b4be23dc2acb22bbe6816818fb47ddd2d9 |