Fixing some test failures (#17852)

* fix controller test

* debug travis failure

* debug some more

* more debugging

* more debugging

* another debug

* another debug

* more debug + fix test?

* remove debug changes

* remove forced failure

* ui test change, check status in assert for more information in test output

* check image magick status AFTER comparison threshold check

* try to debug test performance on travis

* try to speed up some tests

* fix test change

* fix test

* update two screenshots

* update UI test again and fix random failure hopefully

* try to get rid of some more random failures

* some more debugging

* try fixing some more random failures

* stop using screenshotSelector in some tests since it (inexplicably) cant find jquery on the test page + mysqli test

* another ui test tweak

* trigger new build

* try fixing more random failures

* updates expected test file

Co-authored-by: sgiehl <stefan@matomo.org>

10 files changed, 30 insertions, 18 deletions

diff --git a/tests/PHPUnit/Framework/Fixture.php b/tests/PHPUnit/Framework/Fixture.php
index 18f7b68d31..bb5e33a9bd 100644
--- a/tests/PHPUnit/Framework/Fixture.php
+++ b/tests/PHPUnit/Framework/Fixture.php
@@ -395,6 +395,7 @@ class Fixture extends \PHPUnit\Framework\Assert
         Date::$now = null;
         FrontController::$requestId = null;
         Cache::$cache = null;
+        Common::$isCliMode = null;
         MockFileMethods::reset();
         Archive::clearStaticCache();
         DataTableManager::getInstance()->deleteAll();
diff --git a/tests/PHPUnit/Integration/ArchiveTest.php b/tests/PHPUnit/Integration/ArchiveTest.php
index 1e274431d7..d8c6f2b1f4 100644
--- a/tests/PHPUnit/Integration/ArchiveTest.php
+++ b/tests/PHPUnit/Integration/ArchiveTest.php
@@ -28,10 +28,9 @@ use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
 
 class ArchiveTest extends IntegrationTestCase
 {
-    public function setUp(): void
+    protected static function beforeTableDataCached()
     {
-        parent::setUp();
-
+        parent::beforeTableDataCached(); // TODO: Change the autogenerated stub
         Fixture::createWebsite('2014-05-06');
     }
 
diff --git a/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php b/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php
index aedb995fa8..2cdf533cf0 100644
--- a/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php
+++ b/tests/PHPUnit/Integration/DataAccess/LogAggregatorTest.php
@@ -13,6 +13,7 @@ use Piwik\Config;
 use Piwik\Common;
 use Piwik\DataAccess\LogAggregator;
 use Piwik\Date;
+use Piwik\Db;
 use Piwik\Period;
 use Piwik\Segment;
 use Piwik\Site;
diff --git a/tests/PHPUnit/Integration/LogImporterTest.php b/tests/PHPUnit/Integration/LogImporterTest.php
index a83233c792..dd728bcfee 100644
--- a/tests/PHPUnit/Integration/LogImporterTest.php
+++ b/tests/PHPUnit/Integration/LogImporterTest.php
@@ -25,9 +25,10 @@ class LogImporterTest extends IntegrationTestCase
 {
     public static $dateTime = '2012-08-09 11:22:33';
 
-    public function setUp(): void
+    protected static function beforeTableDataCached()
     {
-        parent::setUp();
+        parent::beforeTableDataCached();
+
         Fixture::createSuperUser(false);
         Fixture::createWebsite(self::$dateTime, 0, false, 'https://matomo.org/', 1, null, null, null, null, 0);
         Fixture::createWebsite(self::$dateTime, 1, false, 'https://shop.matomo.org/', 1, null, null, null, null, 0);
diff --git a/tests/PHPUnit/Integration/Tracker/RequestSetTest.php b/tests/PHPUnit/Integration/Tracker/RequestSetTest.php
index 134e65a0e8..b0448e45dc 100644
--- a/tests/PHPUnit/Integration/Tracker/RequestSetTest.php
+++ b/tests/PHPUnit/Integration/Tracker/RequestSetTest.php
@@ -40,9 +40,9 @@ class RequestSetTest extends IntegrationTestCase
     private $post;
     private $time;
 
-    public function setUp(): void
+    protected static function beforeTableDataCached()
     {
-        parent::setUp();
+        parent::beforeTableDataCached();
 
         Fixture::createWebsite('2014-01-01 00:00:00');
         Fixture::createWebsite('2014-01-01 00:00:00', 0, false, 'http://www.example.com');
@@ -50,6 +50,11 @@ class RequestSetTest extends IntegrationTestCase
         foreach (range(3,10) as $idSite) {
             Fixture::createWebsite('2014-01-01 00:00:00');
         }
+    }
+
+    public function setUp(): void
+    {
+        parent::setUp();
 
         $this->requestSet = $this->buildNewRequestSetThatIsNotInitializedYet();
         $this->requestSet->setRequests(array(array('idsite' => 1), array('idsite' => 2)));
diff --git a/tests/PHPUnit/Integration/Tracker/RequestTest.php b/tests/PHPUnit/Integration/Tracker/RequestTest.php
index 80835701cf..796f664dd7 100644
--- a/tests/PHPUnit/Integration/Tracker/RequestTest.php
+++ b/tests/PHPUnit/Integration/Tracker/RequestTest.php
@@ -33,15 +33,20 @@ class RequestTest extends IntegrationTestCase
 
     private $time;
 
-    public function setUp(): void
+    protected static function beforeTableDataCached()
     {
-        parent::setUp();
+        parent::beforeTableDataCached();
 
         Fixture::createWebsite('2014-01-01 00:00:00');
         Fixture::createWebsite('2014-01-01 00:00:00');
         foreach (range(3,14) as $idSite) {
             Fixture::createWebsite('2014-01-01 00:00:00');
         }
+    }
+
+    public function setUp(): void
+    {
+        parent::setUp();
 
         Cache::deleteTrackerCache();
 
diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png b/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png
index e00f9b52fb..1db7657738 100644
--- a/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png
+++ b/tests/UI/expected-screenshots/UIIntegrationTest_admin_diagnostics_configfile.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:21c1bab87ad7710fc5f2a615115906ca899a6b31424bb18d57337606b3d81e68
-size 5032576
+oid sha256:7bd8b73aeb056eda0e86e2f8c20ea40234523cde53f638e8bb4acd02aa70f2dc
+size 5039457
diff --git a/tests/UI/specs/OnlyRawDataNotification_spec.js b/tests/UI/specs/OnlyRawDataNotification_spec.js
index 563acfcf95..41b5081ca1 100644
--- a/tests/UI/specs/OnlyRawDataNotification_spec.js
+++ b/tests/UI/specs/OnlyRawDataNotification_spec.js
@@ -27,7 +27,8 @@ describe("OnlyRawDataNotification", function () {
 
     it("should show notification when only raw data exists", async function () {
         await page.goto(pageUrl);
-        pageWrap = await page.$('#notificationContainer');
-        expect(await pageWrap.screenshot()).to.matchImage('show_notification_when_only_raw_data_exists');
+        await page.waitFor('.widget');
+        const notificationContainer = await page.$('#notificationContainer');
+        expect(await notificationContainer.screenshot()).to.matchImage('show_notification_when_only_raw_data_exists');
     });
 });
diff --git a/tests/lib/screenshot-testing/support/chai-extras.js b/tests/lib/screenshot-testing/support/chai-extras.js
index 52c1b4bbbf..a2aad8d729 100644
--- a/tests/lib/screenshot-testing/support/chai-extras.js
+++ b/tests/lib/screenshot-testing/support/chai-extras.js
@@ -133,10 +133,6 @@ module.exports = function makeChaiImageAssert(comparisonCommand = 'compare') {
                 `the '${comparisonCommand}' command output could not be parsed, should be` +
                 ` an integer, got: ${allOutput.replace(/\s+$/g, '')}`);
 
-            if (result.status !== 0) {
-                return false;
-            }
-
             if (pixelError === 0) {
                 return true;
             }
@@ -154,6 +150,9 @@ module.exports = function makeChaiImageAssert(comparisonCommand = 'compare') {
             // allow a 10 pixel difference only
             chai.assert(pixelError <= 10, `images differ in ${pixelError} pixels (command output: ${allOutput.replace(/\s+$/g, '')})`);
 
+            // if pixel error passes, but status is non-zero for some reason
+            chai.assert(result.status === 0, `the '${comparisonCommand}' command returned a non-zero status: ${result.status}. Output was ${allOutput.replace(/\s+$/g, '')}`);
+
             return true;
         }
 
diff --git a/tests/lib/screenshot-testing/support/page-renderer.js b/tests/lib/screenshot-testing/support/page-renderer.js
index a5f3fea3ce..45cabd8de2 100644
--- a/tests/lib/screenshot-testing/support/page-renderer.js
+++ b/tests/lib/screenshot-testing/support/page-renderer.js
@@ -142,7 +142,7 @@ PageRenderer.prototype.jQuery = async function (selector, options = {}) {
 };
 
 PageRenderer.prototype.screenshotSelector = async function (selector) {
-    await this.waitFor(() => !! window.$);
+    await this.waitFor(() => !! window.$, { timeout: 60000 });
 
     const result = await this.webpage.evaluate(function (selector) {
         window.jQuery('html').addClass('uiTest');