Use newer version of Puppeteer for UI tests (#17880)

* Use latest version of Puppeteer for UI tests

* use newer node on travis

* test

* use waitForTimeout / waitForSelector instead of waitFor

* Handle BC for removed waitFor method

* Updates expected UI test files

* update submodules

* improve/fix some UI tests

* Updates expected UI test files

* improve/fix some UI tests

* update url-parse dependency

* simplify some UI tests

* update screenshot

* avoid using merged assets that are broken

* fix expected compare exit code

* don't use merged assets for TwoFactorAuth UI tests

it seems due to too many requests to login, logout and other pages merging the CoreJS assets somehow fails, and results in returning an empty file. Even the reload fails to serve the correct content, so maybe the file, the content should be written to, is somehow locked. Disabling the merged assets seems to fix this issue for this tests. Locally the tests are also running correctly with merged assets.

* improve handling of automatic css / js reloading in UI tests

* increase timeouts of TwoFactorAuth UI tests

* fix 2fa ui test

* revert change

* update new screenshot

* updates UI screenshots

* submodule updates

18 files changed, 48 insertions, 48 deletions

diff --git a/plugins/PrivacyManager/tests/UI/PrivacyManager_spec.js b/plugins/PrivacyManager/tests/UI/PrivacyManager_spec.js
index 7573d9902b..6e4410867a 100644
--- a/plugins/PrivacyManager/tests/UI/PrivacyManager_spec.js
+++ b/plugins/PrivacyManager/tests/UI/PrivacyManager_spec.js
@@ -23,7 +23,7 @@ describe("PrivacyManager", function () {
     async function setAnonymizeStartEndDate()
     {
         // make sure tests do not fail every day
-        await page.waitFor('input.anonymizeStartDate');
+        await page.waitForSelector('input.anonymizeStartDate');
         await page.evaluate(function () {
             $('input.anonymizeStartDate').val('2018-03-02').change();
             $('input.anonymizeEndDate').val('2018-03-02').change();
@@ -44,7 +44,7 @@ describe("PrivacyManager", function () {
     {
         var elem = await page.jQuery('.modal.open .modal-footer a:contains('+button+')');
         await elem.click();
-        await page.waitFor(500);
+        await page.waitForTimeout(500);
         await page.waitForNetworkIdle();
     }
 
@@ -52,19 +52,19 @@ describe("PrivacyManager", function () {
     {
         await page.click('.findDataSubjects .btn');
         await page.waitForNetworkIdle();
-        await page.waitFor(250);
+        await page.waitForTimeout(250);
     }
 
     async function anonymizePastData()
     {
         await page.click('.anonymizePastData .btn');
-        await page.waitFor(1000); // wait for animation
+        await page.waitForTimeout(1000); // wait for animation
     }
 
     async function deleteDataSubjects()
     {
         await page.evaluate(() => $('.deleteDataSubjects input').click());
-        await page.waitFor(500); // wait for animation
+        await page.waitForTimeout(500); // wait for animation
     }
 
     async function enterSegmentMatchValue(value) {
@@ -84,7 +84,7 @@ describe("PrivacyManager", function () {
         await page.waitForFunction('$("'+selector+'").length > 0');
         var elem = await page.jQuery(selector);
         await elem.click();
-        await page.waitFor(100);
+        await page.waitForTimeout(100);
     }
 
     async function selectActionColumn(title)
@@ -182,7 +182,7 @@ describe("PrivacyManager", function () {
         await loadActionPage('privacySettings');
         await page.click('[name="anonymizeLocation"] label');
         await page.click('[name="anonymizeTheUserId"] label');
-        await page.waitFor(500);
+        await page.waitForTimeout(500);
         await selectActionColumn('time_spent_ref_action');
         await selectActionColumn('idaction_content_name');
 
@@ -192,7 +192,7 @@ describe("PrivacyManager", function () {
     it('should confirm anonymize location and action column', async function() {
         await anonymizePastData();
         await selectModalButton('Yes');
-        await page.waitFor(1000);
+        await page.waitForTimeout(1000);
         await setAnonymizeStartEndDate();
 
         await captureAnonymizeLogData('anonymizelogdata_anonymizelocation_anduserid_and_action_column_confirmed');
@@ -200,7 +200,7 @@ describe("PrivacyManager", function () {
 
     it('should anonymize only one site and different date pre filled', async function() {
         await page.click('.form-group #anonymizeSite .title');
-        await page.waitFor(1000);
+        await page.waitForTimeout(1000);
         await page.click(".form-group #anonymizeSite [title='Site 1']");
         await page.click('[name="anonymizeIp"] label');
         await page.evaluate(function () {
@@ -214,7 +214,7 @@ describe("PrivacyManager", function () {
     it('should anonymize only one site and different date confirmed', async function() {
         await anonymizePastData();
         await selectModalButton('Yes');
-        await page.waitFor(1000);
+        await page.waitForTimeout(1000);
         await setAnonymizeStartEndDate();
 
         await captureAnonymizeLogData('anonymizelogdata_one_site_and_custom_date_confirmed');
@@ -229,7 +229,7 @@ describe("PrivacyManager", function () {
     it('should show no visitor found message', async function() {
         await enterSegmentMatchValue('userfoobar');
         await findDataSubjects();
-        await page.waitFor('.manageGdpr tr');
+        await page.waitForSelector('.manageGdpr tr');
         await page.mouse.move(-10, -10);
 
         await capturePage('gdpr_tools_no_visits_found');
@@ -267,19 +267,19 @@ describe("PrivacyManager", function () {
 
     it('should ask for confirmation before deleting any visit', async function() {
         await deleteDataSubjects();
-        const modal = await page.waitFor('.modal.open', { visible: true });
+        const modal = await page.waitForSelector('.modal.open', { visible: true });
         expect(await modal.screenshot()).to.matchImage('gdpr_tools_delete_visit_unconfirmed');
     });
 
     it('should be able to cancel deletion and not delete any data', async function() {
         await selectModalButton('No');
-        await page.waitFor(500);
+        await page.waitForTimeout(500);
         await capturePage('gdpr_tools_delete_visit_cancelled');
     });
 
     it('should verify really no data deleted', async function() {
         await loadActionPage('gdprTools');
-        await page.waitFor(1000);
+        await page.waitForTimeout(1000);
         await enterSegmentMatchValue('userId203');
         await findDataSubjects();
         await page.click('.entityTable tbody tr:nth-child(2) .checkInclude label');
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_cancelled.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_cancelled.png
index 2a733502b0..1e61e9cf40 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_cancelled.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_cancelled.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:955a5c35636f3f70431b59938387cfa968540e88dceda3cc89e39cf8b4eb8c6c
-size 215966
+oid sha256:638e813c43505b2d715de1e299770b964990fd0c3c18c3788e4ece77aa8bce47
+size 215963
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_confirmed.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_confirmed.png
index 527e87d3e6..2eb399ddc4 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_confirmed.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_confirmed.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4b426466ee070d0fd02306617343ab61ea98ab6a778cea28432e7f5f80290f1a
-size 219332
+oid sha256:06495f98a6b727d1da6bbc84921e6559b9cfcb8290a906d42e46cc9cd57dfcdb
+size 220870
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_prefilled.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_prefilled.png
index 2a733502b0..1e61e9cf40 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_prefilled.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizeip_and_visit_column_prefilled.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:955a5c35636f3f70431b59938387cfa968540e88dceda3cc89e39cf8b4eb8c6c
-size 215966
+oid sha256:638e813c43505b2d715de1e299770b964990fd0c3c18c3788e4ece77aa8bce47
+size 215963
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizelocation_anduserid_and_action_column_confirmed.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizelocation_anduserid_and_action_column_confirmed.png
index 3ba2b8b5b6..cb450b77f4 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizelocation_anduserid_and_action_column_confirmed.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizelocation_anduserid_and_action_column_confirmed.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d96c01eabcee611b916292525c7656a426c4d6f166cd555d7961ee6345f42dbf
-size 231979
+oid sha256:193b062a5a058a6955ce2ef7d66580aa898aee2de16b8484b02cdd68647d0572
+size 233321
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizelocation_anduserid_and_action_column_prefilled.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizelocation_anduserid_and_action_column_prefilled.png
index 83c16ff82a..222188c6db 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizelocation_anduserid_and_action_column_prefilled.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_anonymizelocation_anduserid_and_action_column_prefilled.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e578921ac9da901a299077ef55c9a99201ecc96223912d18efda4ca861b23b9f
-size 228619
+oid sha256:c6da3e4e4b69ca9e75de221171d5993ec26b5eb5debade3fb0b1e9975ca02621
+size 229863
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_one_site_and_custom_date_confirmed.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_one_site_and_custom_date_confirmed.png
index a882c9a832..5187405a6d 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_one_site_and_custom_date_confirmed.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_one_site_and_custom_date_confirmed.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ce0b972bae27b789467196a74d1311987e97de3a53d45f4bcd02da3865dcb4fc
-size 240513
+oid sha256:87084a49a3174ba506c19cebb0fc3a6ce91ce26836e79c9db1c88e9d3edcb56b
+size 241565
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_one_site_and_custom_date_prefilled.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_one_site_and_custom_date_prefilled.png
index 30f41c3473..77336c5088 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_one_site_and_custom_date_prefilled.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_anonymizelogdata_one_site_and_custom_date_prefilled.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:bcbe5429259d3fdd0b26e3bdc78e826600d79bcd0f4e07babf201cb231126688
-size 231567
+oid sha256:631206c6bba4d1c65179c171de3f34aed6efab4d05f9f7008385576adbc1eaa0
+size 232923
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_default.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_default.png
index 8a01ce40eb..d04e6e7362 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_default.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_default.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b05186a554001b0ac13caf51778acb72565b2955478f20e2486481d47ed2ca0a
-size 62774
+oid sha256:0dc061653083632c413a46a669eaf8d5a30b743f60a6ed3ff6b2cd334e06df85
+size 62462
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled.png
index 6e8392eecb..1bccf2b8ba 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8931e34b2f9c591e7b293a0fd1c3078bdbd3d081d1cb6e119b024a71fc96ddf
-size 581083
+oid sha256:29f7a8a10f4ae63d3cddf9c80644378e7625d1d17b30986503444c6a2111db89
+size 567547
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled_verified_no_data_deleted.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled_verified_no_data_deleted.png
index 6216e88f41..f5d266a55a 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled_verified_no_data_deleted.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_cancelled_verified_no_data_deleted.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:323b04402124efac3c47a76fb178d05a2989ffdf3fceb7d780096160720feb84
-size 574962
+oid sha256:c96c2d345f869408f4e5e346577b7d9245cbe961c651810cee2811606b92103e
+size 561835
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_confirmed.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_confirmed.png
index 788f1fce8d..f8b86a7ce9 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_confirmed.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_delete_visit_confirmed.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:86f7d76734614cd895bed0e973bb469a50d936c704260f94b328e5055a061d43
-size 129652
+oid sha256:f91d4fea1087cfcf6ab7683fe71831cbd869db1aff9fdc16e9655e29a2da2c71
+size 129007
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_enrich_segment_by_ip.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_enrich_segment_by_ip.png
index a5c0c71d0f..911bd61f01 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_enrich_segment_by_ip.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_enrich_segment_by_ip.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:42452acb7ad8a17c73f51eb3b7d49bbd33f1366d546faab54ee2373f25a870db
-size 581465
+oid sha256:6b3f74a93a0d0ce9a458f08b79106edec939a20403a9730ced8a8f2c7677e2fe
+size 567895
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_no_visits_found.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_no_visits_found.png
index bd47680816..8898412f55 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_no_visits_found.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_no_visits_found.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:148476ea54401b0a3ba9a6e2eb3a8c27d912f029fd44dc33813fc35dce151e55
-size 68531
+oid sha256:2d5c09c53ebcd95563f4675f34f6b723004f41ba9fbc642f79b90f94f212edf8
+size 68186
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_uncheck_one_visit.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_uncheck_one_visit.png
index 6e8392eecb..1bccf2b8ba 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_uncheck_one_visit.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_uncheck_one_visit.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a8931e34b2f9c591e7b293a0fd1c3078bdbd3d081d1cb6e119b024a71fc96ddf
-size 581083
+oid sha256:29f7a8a10f4ae63d3cddf9c80644378e7625d1d17b30986503444c6a2111db89
+size 567547
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_found.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_found.png
index dcda22afde..218292e2fd 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_found.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_gdpr_tools_visits_found.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:03fb293c3e477a3e51c966aa5d9a401e95c124b531b1398bc5336a75a0a06467
-size 575315
+oid sha256:81fde2b193c87cbd81b092e8d8509ad12aa7cef5efe6c131c6d6ed48f9e3c730
+size 562200
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_privacy_settings_default.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_privacy_settings_default.png
index c04fc1e707..24026075be 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_privacy_settings_default.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_privacy_settings_default.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7e2a05a2210124164554166a5a381cb1cb4cea7889c2e6efae56c24602302f0f
-size 545190
+oid sha256:39b338e165b7fe1ef7333d1ad3a2be3285fb5a899e1d29eb4dc5fbd4b4daa700
+size 545154
diff --git a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_users_opt_out_default.png b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_users_opt_out_default.png
index 89a9410ba8..656a98e691 100644
--- a/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_users_opt_out_default.png
+++ b/plugins/PrivacyManager/tests/UI/expected-screenshots/PrivacyManager_users_opt_out_default.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:20306dce97d09ad756083c0cf55d53b02ea44023f68c7a3fad8dddc151c9b499
-size 154394
+oid sha256:32f3b9b6be11da85e702b0c40d3dcab5a0b41b5e627bd6b62b4773365e752807
+size 154252