Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/matomo-org/matomo.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--core/Archive/ArchivePurger.php58
-rw-r--r--core/CronArchive.php11
-rw-r--r--core/DataAccess/Model.php35
-rw-r--r--core/Scheduler/Scheduler.php18
-rw-r--r--core/Scheduler/Timetable.php6
m---------misc/log-analytics0
-rw-r--r--plugins/CoreAdminHome/API.php16
-rw-r--r--plugins/CoreAdminHome/Tasks.php41
m---------plugins/QueuedTracking0
m---------plugins/SecurityInfo0
m---------plugins/TasksTimetable0
m---------plugins/VisitorGenerator0
12 files changed, 137 insertions, 48 deletions
diff --git a/core/Archive/ArchivePurger.php b/core/Archive/ArchivePurger.php
index e41ce375c5..910a539dbb 100644
--- a/core/Archive/ArchivePurger.php
+++ b/core/Archive/ArchivePurger.php
@@ -10,12 +10,13 @@ namespace Piwik\Archive;
use Piwik\ArchiveProcessor\Rules;
use Piwik\Config;
+use Piwik\Container\StaticContainer;
use Piwik\DataAccess\ArchiveTableCreator;
use Piwik\DataAccess\Model;
use Piwik\Date;
use Piwik\Db;
-use Piwik\Log;
use Piwik\Piwik;
+use Psr\Log\LoggerInterface;
/**
* Service that purges temporary, error-ed, invalid and custom range archives from archive tables.
@@ -64,7 +65,12 @@ class ArchivePurger
*/
private $now;
- public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null)
+ /**
+ * @var LoggerInterface
+ */
+ private $logger;
+
+ public function __construct(Model $model = null, Date $purgeCustomRangesOlderThan = null, LoggerInterface $logger = null)
{
$this->model = $model ?: new Model();
@@ -73,6 +79,7 @@ class ArchivePurger
$this->yesterday = Date::factory('yesterday');
$this->today = Date::factory('today');
$this->now = time();
+ $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
}
/**
@@ -90,15 +97,25 @@ class ArchivePurger
// the constraint will hit an INDEX and speed up the inner join that happens in getInvalidatedArchiveIdsSafeToDelete().
$idSites = $this->model->getSitesWithInvalidatedArchive($numericTable);
if (empty($idSites)) {
+ $this->logger->info("No sites with invalidated archives found in {table}.", array('table' => $numericTable));
return;
}
$archiveIds = $this->model->getInvalidatedArchiveIdsSafeToDelete($numericTable, $idSites);
if (empty($archiveIds)) {
+ $this->logger->info("No invalidated archives found in {table} with newer, valid archives.", array('table' => $numericTable));
return;
}
- $this->deleteArchiveIds($date, $archiveIds);
+ $this->logger->info("Found {countArchiveIds} invalidated archives safe to delete in {table}.", array(
+ 'table' => $numericTable, 'countArchiveIds' => count($archiveIds)
+ ));
+
+ $deletedRowCount = $this->deleteArchiveIds($date, $archiveIds);
+
+ $this->logger->info("Deleted {count} rows in {table} and its associated blob table.", array(
+ 'table' => $numericTable, 'count' => $deletedRowCount
+ ));
}
/**
@@ -113,13 +130,21 @@ class ArchivePurger
$idArchivesToDelete = $this->getOutdatedArchiveIds($dateStart, $purgeArchivesOlderThan);
if (!empty($idArchivesToDelete)) {
- $this->deleteArchiveIds($dateStart, $idArchivesToDelete);
+ $deletedRowCount = $this->deleteArchiveIds($dateStart, $idArchivesToDelete);
+
+ $this->logger->info("Deleted {count} rows in archive tables (numeric + blob) for {date}.", array(
+ 'count' => $deletedRowCount,
+ 'date' => $dateStart
+ ));
+ } else {
+ $this->logger->info("No outdated archives found in archive numeric table for {date}.", array('date' => $dateStart));
}
- Log::debug("Purging temporary archives: done [ purged archives older than %s in %s ] [Deleted IDs: %s]",
- $purgeArchivesOlderThan,
- $dateStart->toString("Y-m"),
- implode(',', $idArchivesToDelete));
+ $this->logger->debug("Purging temporary archives: done [ purged archives older than {date} in {yearMonth} ] [Deleted IDs: {deletedIds}]", array(
+ 'date' => $purgeArchivesOlderThan,
+ 'yearMonth' => $dateStart->toString('Y-m'),
+ 'deletedIds' => implode(',', $idArchivesToDelete)
+ ));
}
protected function getOutdatedArchiveIds(Date $date, $purgeArchivesOlderThan)
@@ -148,10 +173,16 @@ class ArchivePurger
$numericTable = ArchiveTableCreator::getNumericTable($date);
$blobTable = ArchiveTableCreator::getBlobTable($date);
- $this->model->deleteArchivesWithPeriod($numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan);
+ $deletedCount = $this->model->deleteArchivesWithPeriod(
+ $numericTable, $blobTable, Piwik::$idPeriods['range'], $this->purgeCustomRangesOlderThan);
+
+ $this->logger->info("Purged {count} range archive rows from {numericTable} & {blobTable}.", array(
+ 'count' => $deletedCount,
+ 'numericTable' => $numericTable,
+ 'blobTable' => $blobTable
+ ));
- Log::debug("Purging Custom Range archives: done [ purged archives older than %s from %s / blob ]",
- $this->purgeCustomRangesOlderThan, $numericTable);
+ $this->logger->debug(" [ purged archives older than {threshold} ]", array('threshold' => $this->purgeCustomRangesOlderThan));
}
/**
@@ -159,6 +190,7 @@ class ArchivePurger
*
* @param Date $date
* @param $idArchivesToDelete
+ * @return int Number of rows deleted from both numeric + blob table.
*/
protected function deleteArchiveIds(Date $date, $idArchivesToDelete)
{
@@ -166,9 +198,11 @@ class ArchivePurger
$numericTable = ArchiveTableCreator::getNumericTable($date);
$blobTable = ArchiveTableCreator::getBlobTable($date);
+ $deletedCount = 0;
foreach ($batches as $idsToDelete) {
- $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
+ $deletedCount += $this->model->deleteArchiveIds($numericTable, $blobTable, $idsToDelete);
}
+ return $deletedCount;
}
/**
diff --git a/core/CronArchive.php b/core/CronArchive.php
index 50716ea756..82b7e2922b 100644
--- a/core/CronArchive.php
+++ b/core/CronArchive.php
@@ -429,13 +429,12 @@ class CronArchive
$this->log("Starting Scheduled tasks... ");
- $tasksOutput = $this->request("?module=API&method=CoreAdminHome.runScheduledTasks&format=csv&convertToUnicode=0&token_auth=" . $this->token_auth);
+ API\Request::processRequest("CoreAdminHome.runScheduledTasks", array(
+ 'token_auth' => $this->token_auth,
+ 'format' => 'original', // so exceptions get thrown
+ 'trigger' => 'archivephp'
+ ));
- if ($tasksOutput == \Piwik\DataTable\Renderer\Csv::NO_DATA_AVAILABLE) {
- $tasksOutput = " No task to run";
- }
-
- $this->log($tasksOutput);
$this->log("done");
$this->logSection("");
}
diff --git a/core/DataAccess/Model.php b/core/DataAccess/Model.php
index 0980983fe3..8df49d16a3 100644
--- a/core/DataAccess/Model.php
+++ b/core/DataAccess/Model.php
@@ -10,9 +10,11 @@ namespace Piwik\DataAccess;
use Exception;
use Piwik\Common;
+use Piwik\Container\StaticContainer;
use Piwik\Db;
use Piwik\DbHelper;
use Piwik\Sequence;
+use Psr\Log\LoggerInterface;
/**
* Cleans up outdated archives
@@ -21,6 +23,15 @@ use Piwik\Sequence;
*/
class Model
{
+ /**
+ * @var LoggerInterface
+ */
+ private $logger;
+
+ public function __construct(LoggerInterface $logger = null)
+ {
+ $this->logger = $logger ?: StaticContainer::get('Psr\Log\LoggerInterface');
+ }
/**
* Returns the archives IDs that have already been invalidated and have been since re-processed.
@@ -118,13 +129,21 @@ class Model
$query = "DELETE FROM %s WHERE period = ? AND ts_archived < ?";
$bind = array($period, $date);
- Db::query(sprintf($query, $numericTable), $bind);
+ $queryObj = Db::query(sprintf($query, $numericTable), $bind);
+ $deletedRows = $queryObj->rowCount();
try {
- Db::query(sprintf($query, $blobTable), $bind);
+ $queryObj = Db::query(sprintf($query, $blobTable), $bind);
+ $deletedRows += $queryObj->rowCount();
} catch (Exception $e) {
// Individual blob tables could be missing
+ $this->logger->debug("Unable to delete archives by period from {blobTable}.", array(
+ 'blobTable' => $blobTable,
+ 'exception' => $e
+ ));
}
+
+ return $deletedRows;
}
public function deleteArchiveIds($numericTable, $blobTable, $idsToDelete)
@@ -132,13 +151,21 @@ class Model
$idsToDelete = array_values($idsToDelete);
$query = "DELETE FROM %s WHERE idarchive IN (" . Common::getSqlStringFieldsArray($idsToDelete) . ")";
- Db::query(sprintf($query, $numericTable), $idsToDelete);
+ $queryObj = Db::query(sprintf($query, $numericTable), $idsToDelete);
+ $deletedRows = $queryObj->rowCount();
try {
- Db::query(sprintf($query, $blobTable), $idsToDelete);
+ $queryObj = Db::query(sprintf($query, $blobTable), $idsToDelete);
+ $deletedRows += $queryObj->rowCount();
} catch (Exception $e) {
// Individual blob tables could be missing
+ $this->logger->debug("Unable to delete archive IDs from {blobTable}.", array(
+ 'blobTable' => $blobTable,
+ 'exception' => $e
+ ));
}
+
+ return $deletedRows;
}
public function getArchiveIdAndVisits($numericTable, $idSite, $period, $dateStartIso, $dateEndIso, $minDatetimeIsoArchiveProcessedUTC, $doneFlags, $doneFlagValues)
diff --git a/core/Scheduler/Scheduler.php b/core/Scheduler/Scheduler.php
index 7156e1be9c..f4e2097ca5 100644
--- a/core/Scheduler/Scheduler.php
+++ b/core/Scheduler/Scheduler.php
@@ -99,9 +99,9 @@ class Scheduler
// for every priority level, starting with the highest and concluding with the lowest
$executionResults = array();
- for ($priority = Task::HIGHEST_PRIORITY;
- $priority <= Task::LOWEST_PRIORITY;
- ++$priority) {
+ for ($priority = Task::HIGHEST_PRIORITY; $priority <= Task::LOWEST_PRIORITY; ++$priority) {
+ $this->logger->debug("Executing tasks with priority {priority}:", array('priority' => $priority));
+
// loop through each task
foreach ($tasks as $task) {
// if the task does not have the current priority level, don't execute it yet
@@ -113,14 +113,24 @@ class Scheduler
$shouldExecuteTask = $this->timetable->shouldExecuteTask($taskName);
if ($this->timetable->taskShouldBeRescheduled($taskName)) {
- $this->timetable->rescheduleTask($task);
+ $rescheduledDate = $this->timetable->rescheduleTask($task);
+
+ $this->logger->debug("Task {task} is scheduled to run again for {date}.", array('task' => $taskName, 'date' => $rescheduledDate));
}
if ($shouldExecuteTask) {
+ $this->logger->info("Scheduler: executing task {taskName}...", array('taskName' => $taskName));
+
+ $timer = new Timer();
+
$this->isRunningTask = true;
$message = $this->executeTask($task);
$this->isRunningTask = false;
+ $this->logger->info("Scheduler: finished executing task {taskName}. {timeElapsed}", array(
+ 'taskName' => $taskName, 'timeElapsed' => $timer
+ ));
+
$executionResults[] = array('task' => $taskName, 'output' => $message);
}
}
diff --git a/core/Scheduler/Timetable.php b/core/Scheduler/Timetable.php
index 5c9bc69696..c3c01ab167 100644
--- a/core/Scheduler/Timetable.php
+++ b/core/Scheduler/Timetable.php
@@ -105,9 +105,13 @@ class Timetable
public function rescheduleTask(Task $task)
{
+ $rescheduledTime = $task->getRescheduledTime();
+
// update the scheduled time
- $this->timetable[$task->getName()] = $task->getRescheduledTime();
+ $this->timetable[$task->getName()] = $rescheduledTime;
$this->save();
+
+ return Date::factory($rescheduledTime);
}
public function save()
diff --git a/misc/log-analytics b/misc/log-analytics
-Subproject 05dba484f1933a710f171de9b169ee03d240710
+Subproject a4d1eea78e6ea52b1b72e63a494f02e9d6f7223
diff --git a/plugins/CoreAdminHome/API.php b/plugins/CoreAdminHome/API.php
index 6329fc8993..2b809b792a 100644
--- a/plugins/CoreAdminHome/API.php
+++ b/plugins/CoreAdminHome/API.php
@@ -15,6 +15,7 @@ use Piwik\Db;
use Piwik\Piwik;
use Piwik\Scheduler\Scheduler;
use Piwik\Site;
+use Psr\Log\LoggerInterface;
/**
* @method static \Piwik\Plugins\CoreAdminHome\API getInstance()
@@ -22,6 +23,16 @@ use Piwik\Site;
class API extends \Piwik\Plugin\API
{
/**
+ * @var Scheduler
+ */
+ private $scheduler;
+
+ public function __construct(Scheduler $scheduler)
+ {
+ $this->scheduler = $scheduler;
+ }
+
+ /**
* Will run all scheduled tasks due to run at this time.
*
* @return array
@@ -31,10 +42,7 @@ class API extends \Piwik\Plugin\API
{
Piwik::checkUserHasSuperUserAccess();
- /** @var Scheduler $scheduler */
- $scheduler = StaticContainer::getContainer()->get('Piwik\Scheduler\Scheduler');
-
- return $scheduler->run();
+ return $this->scheduler->run();
}
/**
diff --git a/plugins/CoreAdminHome/Tasks.php b/plugins/CoreAdminHome/Tasks.php
index 700c7d69c2..7a3ff406b5 100644
--- a/plugins/CoreAdminHome/Tasks.php
+++ b/plugins/CoreAdminHome/Tasks.php
@@ -14,9 +14,8 @@ use Piwik\Container\StaticContainer;
use Piwik\DataAccess\ArchiveTableCreator;
use Piwik\Date;
use Piwik\Db;
-use Piwik\Log;
use Piwik\Plugins\CoreAdminHome\Tasks\ArchivesToPurgeDistributedList;
-use Piwik\SettingsServer;
+use Psr\Log\LoggerInterface;
class Tasks extends \Piwik\Plugin\Tasks
{
@@ -25,9 +24,15 @@ class Tasks extends \Piwik\Plugin\Tasks
*/
private $archivePurger;
- public function __construct(ArchivePurger $archivePurger = null)
+ /**
+ * @var LoggerInterface
+ */
+ private $logger;
+
+ public function __construct(ArchivePurger $archivePurger, LoggerInterface $logger)
{
- $this->archivePurger = $archivePurger ?: new ArchivePurger();
+ $this->archivePurger = $archivePurger;
+ $this->logger = $logger;
}
public function schedule()
@@ -44,16 +49,14 @@ class Tasks extends \Piwik\Plugin\Tasks
public function purgeOutdatedArchives()
{
- $logger = StaticContainer::get('Psr\Log\LoggerInterface');
-
if ($this->willPurgingCausePotentialProblemInUI()) {
- $logger->info("Purging temporary archives: skipped (browser triggered archiving not enabled & not running after core:archive)");
+ $this->logger->info("Purging temporary archives: skipped (browser triggered archiving not enabled & not running after core:archive)");
return false;
}
$archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
- $logger->info("Purging archives in {tableCount} archive tables.", array('tableCount' => count($archiveTables)));
+ $this->logger->info("Purging archives in {tableCount} archive tables.", array('tableCount' => count($archiveTables)));
// keep track of dates we purge for, since getTablesArchivesInstalled() will return numeric & blob
// tables (so dates will appear two times, and we should only purge once per date)
@@ -64,15 +67,19 @@ class Tasks extends \Piwik\Plugin\Tasks
list($year, $month) = explode('_', $date);
// Somehow we may have archive tables created with older dates, prevent exception from being thrown
- if ($year > 1990
- && empty($datesPurged[$date])
- ) {
- $dateObj = Date::factory("$year-$month-15");
-
- $this->archivePurger->purgeOutdatedArchives($dateObj);
- $this->archivePurger->purgeArchivesWithPeriodRange($dateObj);
-
- $datesPurged[$date] = true;
+ if ($year > 1990) {
+ if (empty($datesPurged[$date])) {
+ $dateObj = Date::factory("$year-$month-15");
+
+ $this->archivePurger->purgeOutdatedArchives($dateObj);
+ $this->archivePurger->purgeArchivesWithPeriodRange($dateObj);
+
+ $datesPurged[$date] = true;
+ } else {
+ $this->logger->debug("Date {date} already purged.", array('date' => $date));
+ }
+ } else {
+ $this->logger->info("Skipping purging of archive tables *_{year}_{month}, year <= 1990.", array('year' => $year, 'month' => $month));
}
}
}
diff --git a/plugins/QueuedTracking b/plugins/QueuedTracking
-Subproject bc39d75e4c9df58c865ae44f6eae23676e7baf9
+Subproject f1320c0d103942a4a07e6b724a32c17f0b8c6cf
diff --git a/plugins/SecurityInfo b/plugins/SecurityInfo
-Subproject 85a04e6f1a4b6da0ea821ba2827dd093bbc1277
+Subproject 101aa27943576a3d666ddad24710af2e4c46306
diff --git a/plugins/TasksTimetable b/plugins/TasksTimetable
-Subproject a0f38fec7cf905d34059ec1b6a207156adf7579
+Subproject 15197bb051c3c2281d34365d96cc761e621cef9
diff --git a/plugins/VisitorGenerator b/plugins/VisitorGenerator
-Subproject 34d796330e256e947ff976f57e350c775dc061d
+Subproject a1626b4be23dc2acb22bbe6816818fb47ddd2d9
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-31 06:05:20 +0300