diff options
| -rw-r--r-- | config/environment/test.php | 2 | ||||
| -rw-r--r-- | core/CliMulti/Process.php | 16 | ||||
| -rw-r--r-- | core/CliMulti/RequestCommand.php | 2 | ||||
| -rw-r--r-- | core/CronArchive.php | 12 | ||||
| -rw-r--r-- | plugins/CoreConsole/tests/System/ArchiveCronTest.php | 36 | ||||
| -rw-r--r-- | tests/PHPUnit/Integration/CronArchiveTest.php | 32 |
6 files changed, 71 insertions, 29 deletions
diff --git a/config/environment/test.php b/config/environment/test.php index 63af43d95b..e6cbbd62f0 100644 --- a/config/environment/test.php +++ b/config/environment/test.php @@ -134,4 +134,6 @@ return array( file_put_contents($outputFile, json_encode($outputContents)); })), )), + + 'test.vars.forceCliMultiViaCurl' => false, ); diff --git a/core/CliMulti/Process.php b/core/CliMulti/Process.php index ad1d30cd56..8133da8878 100644 --- a/core/CliMulti/Process.php +++ b/core/CliMulti/Process.php @@ -8,6 +8,7 @@ namespace Piwik\CliMulti; use Piwik\CliMulti; +use Piwik\Container\StaticContainer; use Piwik\Filesystem; use Piwik\SettingsServer; @@ -45,6 +46,15 @@ class Process $this->markAsNotStarted(); } + private static function isForcingAsyncProcessMode() + { + try { + return (bool) StaticContainer::get('test.vars.forceCliMultiViaCurl'); + } catch (\Exception $ex) { + return false; + } + } + public function getPid() { return $this->pid; @@ -178,6 +188,12 @@ class Process public static function isSupported() { + if (defined('PIWIK_TEST_MODE') + && self::isForcingAsyncProcessMode() + ) { + return false; + } + if (SettingsServer::isWindows()) { return false; } diff --git a/core/CliMulti/RequestCommand.php b/core/CliMulti/RequestCommand.php index f2e07dcb94..8dbd8d9ed0 100644 --- a/core/CliMulti/RequestCommand.php +++ b/core/CliMulti/RequestCommand.php @@ -103,6 +103,8 @@ class RequestCommand extends ConsoleCommand Url::setHost($hostname); $query = $input->getArgument('url-query'); + $_SERVER['QUERY_STRING'] = $query; + $query = UrlHelper::getArrayFromQueryString($query); // NOTE: this method can create the StaticContainer now foreach ($query as $name => $value) { $_GET[$name] = urldecode($value); diff --git a/core/CronArchive.php b/core/CronArchive.php index da8236b355..57e6ea259d 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -212,6 +212,11 @@ class CronArchive private $cliMultiRequestParser; /** + * @var bool|mixed + */ + private $supportsAsync; + + /** * Constructor. * * @param string|null $processNewSegmentsFrom When to archive new segments from. See [General] process_new_segments_from @@ -235,7 +240,8 @@ class CronArchive $this->rawLogDao = new RawLogDao(); - $this->cliMultiRequestParser = new RequestParser($this->makeCliMulti()->supportsAsync()); + $this->supportsAsync = $this->makeCliMulti()->supportsAsync(); + $this->cliMultiRequestParser = new RequestParser($this->supportsAsync); $this->archiveFilter = new ArchiveFilter(); } @@ -654,7 +660,7 @@ class CronArchive { $request = "?module=API&method=CoreAdminHome.archiveReports&idSite=$idSite&period=$period&date=" . $date . "&format=json"; if ($segment) { - $request .= '&segment=' . urlencode(urlencode($segment)); + $request .= '&segment=' . urlencode($segment); } if (!empty($plugin)) { $request .= "&plugin=" . $plugin; @@ -696,7 +702,7 @@ class CronArchive if (empty($response)) { $message .= "The response was empty. This usually means a server error. A solution to this error is generally to increase the value of 'memory_limit' in your php.ini file. "; - if($this->makeCliMulti()->supportsAsync()) { + if($this->supportsAsync) { $message .= " For more information and the error message please check in your PHP CLI error log file. As this core:archive command triggers PHP processes over the CLI, you can find where PHP CLI logs are stored by running this command: php -i | grep error_log"; } else { $message .= " For more information and the error message please check your web server's error Log file. As this core:archive command triggers PHP processes over HTTP, you can find the error message in your Matomo's web server error logs. "; diff --git a/plugins/CoreConsole/tests/System/ArchiveCronTest.php b/plugins/CoreConsole/tests/System/ArchiveCronTest.php index 034b58906a..7886614f95 100644 --- a/plugins/CoreConsole/tests/System/ArchiveCronTest.php +++ b/plugins/CoreConsole/tests/System/ArchiveCronTest.php @@ -9,6 +9,7 @@ namespace Piwik\Plugins\CoreConsole\tests\System; use Piwik\CronArchive; use Piwik\Plugins\SegmentEditor\API; +use Piwik\Tests\Framework\TestingEnvironmentVariables; use Psr\Container\ContainerInterface; use Piwik\Archive\ArchiveInvalidator; use Piwik\Common; @@ -178,17 +179,18 @@ class ArchiveCronTest extends SystemTestCase // empty the list so nothing is invalidated during core:archive (so we only archive ExamplePlugin and not all plugins) $invalidator->forgetRememberedArchivedReportsToInvalidate(1, Date::factory('2007-04-05')); - $output = $this->runArchivePhpCron(); - - Option::delete(CronArchive::OPTION_ARCHIVING_FINISHED_TS); // clear so segment re-archive logic runs on this run - Option::delete(CronArchive::CRON_INVALIDATION_TIME_OPTION_NAME); - $output = $this->runArchivePhpCron(); // have to run twice since we manually invalidate above + $this->runArchivePhpCron(); // add new segment w/ edited created/edit time so it will not trigger segment re-archiving, then track a visit - // so the segments will be archived w/ other invalidation - self::addNewSegmentToPast(); - self::trackVisitsForToday(); - $output = $this->runArchivePhpCron(); + // so the segments will be archived w/ other invalidation. this also runs core:archive forcing CURL requests. + try { + self::forceCurlCliMulti(); + self::addNewSegmentToPast(); + self::trackVisitsForToday(); + $output = $this->runArchivePhpCron(); + } finally { + self::undoForceCurlCliMulti(); + } $expectedInvalidations = []; $invalidationEntries = $this->getInvalidatedArchiveTableEntries(); @@ -230,7 +232,7 @@ class ArchiveCronTest extends SystemTestCase $invalidator = StaticContainer::get(ArchiveInvalidator::class); $invalidator->markArchivesAsInvalidated([1], ['2007-04-05'], 'day', new Segment('', [1]), false, false, 'ExamplePlugin.ExamplePlugin_example_metric2'); - $output = $this->runArchivePhpCron(['-vvv' => null]); + $output = $this->runArchivePhpCron(); Option::delete(CronArchive::OPTION_ARCHIVING_FINISHED_TS); // clear so segment re-archive logic runs on this run Option::delete(CronArchive::CRON_INVALIDATION_TIME_OPTION_NAME); @@ -358,6 +360,20 @@ class ArchiveCronTest extends SystemTestCase { return Db::fetchAll("SELECT idinvalidation, idarchive, idsite, date1, date2, period, name, status FROM " . Common::prefixTable('archive_invalidations')); } + + private static function undoForceCurlCliMulti() + { + $testVars = new TestingEnvironmentVariables(); + $testVars->forceCliMultiViaCurl = 0; + $testVars->save(); + } + + private static function forceCurlCliMulti() + { + $testVars = new TestingEnvironmentVariables(); + $testVars->forceCliMultiViaCurl = 1; + $testVars->save(); + } } ArchiveCronTest::$fixture = new ManySitesImportedLogs(); diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php index 4d1f489553..041e2e8355 100644 --- a/tests/PHPUnit/Integration/CronArchiveTest.php +++ b/tests/PHPUnit/Integration/CronArchiveTest.php @@ -700,31 +700,31 @@ Done invalidating Processing invalidation: [idinvalidation = 5, idsite = 1, period = day(2019-12-12 - 2019-12-12), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. Processing invalidation: [idinvalidation = 14, idsite = 1, period = day(2019-12-11 - 2019-12-11), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. Processing invalidation: [idinvalidation = 17, idsite = 1, period = day(2019-12-10 - 2019-12-10), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. -Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-12&format=json&segment=actions%253E%253D2&trigger=archivephp -Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-11&format=json&segment=actions%253E%253D2&trigger=archivephp -Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-10&format=json&segment=actions%253E%253D2&trigger=archivephp -Archived website id 1, period = day, date = 2019-12-12, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Archived website id 1, period = day, date = 2019-12-11, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Archived website id 1, period = day, date = 2019-12-10, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-12&format=json&segment=actions%3E%3D2&trigger=archivephp +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-11&format=json&segment=actions%3E%3D2&trigger=archivephp +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-10&format=json&segment=actions%3E%3D2&trigger=archivephp +Archived website id 1, period = day, date = 2019-12-12, segment = 'actions>=2', 0 visits found. Time elapsed: %fs +Archived website id 1, period = day, date = 2019-12-11, segment = 'actions>=2', 0 visits found. Time elapsed: %fs +Archived website id 1, period = day, date = 2019-12-10, segment = 'actions>=2', 0 visits found. Time elapsed: %fs Processing invalidation: [idinvalidation = 6, idsite = 1, period = week(2019-12-09 - 2019-12-15), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. Processing invalidation: [idinvalidation = 22, idsite = 1, period = day(2019-12-02 - 2019-12-02), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. -Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-09&format=json&segment=actions%253E%253D2&trigger=archivephp -Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-02&format=json&segment=actions%253E%253D2&trigger=archivephp -Archived website id 1, period = week, date = 2019-12-09, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs -Archived website id 1, period = day, date = 2019-12-02, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-09&format=json&segment=actions%3E%3D2&trigger=archivephp +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=day&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp +Archived website id 1, period = week, date = 2019-12-09, segment = 'actions>=2', 0 visits found. Time elapsed: %fs +Archived website id 1, period = day, date = 2019-12-02, segment = 'actions>=2', 0 visits found. Time elapsed: %fs Processing invalidation: [idinvalidation = 23, idsite = 1, period = week(2019-12-02 - 2019-12-08), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. -Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-02&format=json&segment=actions%253E%253D2&trigger=archivephp -Archived website id 1, period = week, date = 2019-12-02, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=week&date=2019-12-02&format=json&segment=actions%3E%3D2&trigger=archivephp +Archived website id 1, period = week, date = 2019-12-02, segment = 'actions>=2', 0 visits found. Time elapsed: %fs Processing invalidation: [idinvalidation = 7, idsite = 1, period = month(2019-12-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. -Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=month&date=2019-12-01&format=json&segment=actions%253E%253D2&trigger=archivephp -Archived website id 1, period = month, date = 2019-12-01, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=month&date=2019-12-01&format=json&segment=actions%3E%3D2&trigger=archivephp +Archived website id 1, period = month, date = 2019-12-01, segment = 'actions>=2', 0 visits found. Time elapsed: %fs Processing invalidation: [idinvalidation = 8, idsite = 1, period = year(2019-01-01 - 2019-12-31), name = donee0512c03f7c20af6ef96a8d792c6bb9f, segment = actions>=2]. No next invalidated archive. -Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=year&date=2019-01-01&format=json&segment=actions%253E%253D2&trigger=archivephp -Archived website id 1, period = year, date = 2019-01-01, segment = 'actions%3E%3D2', 0 visits found. Time elapsed: %fs +Starting archiving for ?module=API&method=CoreAdminHome.archiveReports&idSite=1&period=year&date=2019-01-01&format=json&segment=actions%3E%3D2&trigger=archivephp +Archived website id 1, period = year, date = 2019-01-01, segment = 'actions>=2', 0 visits found. Time elapsed: %fs No next invalidated archive. Finished archiving for site 1, 8 API requests, Time elapsed: %fs [1 / 1 done] No more sites left to archive, stopping. |