Merge pull request #11124 from piwik/3.x-dev3.0.1-b2

3.0.1-b2

22 files changed, 226 insertions, 34 deletions

diff --git a/core/DataTable/Filter/ColumnDelete.php b/core/DataTable/Filter/ColumnDelete.php
index 37f0b1bf97..18ba41a4b2 100644
--- a/core/DataTable/Filter/ColumnDelete.php
+++ b/core/DataTable/Filter/ColumnDelete.php
@@ -175,7 +175,8 @@ class ColumnDelete extends BaseFilter
                 unset($table[$index][$column]);
             }
 
-            $this->removeColumnsFromTable($row);
+            // Restore me in Piwik 4
+            //$this->removeColumnsFromTable($row);
         }
     }
 
diff --git a/core/FileIntegrity.php b/core/FileIntegrity.php
index b457f1d05f..6bc479e8f5 100644
--- a/core/FileIntegrity.php
+++ b/core/FileIntegrity.php
@@ -41,6 +41,9 @@ class FileIntegrity
             );
         }
 
+
+        $messages = self::getMessagesDirectoriesFoundButNotExpected($messages);
+
         $messages = self::getMessagesFilesFoundButNotExpected($messages);
 
         $messages = self::getMessagesFilesMismatch($messages);
@@ -54,11 +57,13 @@ class FileIntegrity
     protected static function getFilesNotInManifestButExpectedAnyway()
     {
         return array(
-            '*/.htaccess',
-            '*/web.config',
+            '*.htaccess',
+            '*web.config',
             'bootstrap.php',
             'favicon.ico',
             'robots.txt',
+            '.bowerrc',
+            '.phpstorm.meta.php',
             'config/config.ini.php',
             'config/common.ini.php',
             'config/*.config.ini.php',
@@ -66,14 +71,56 @@ class FileIntegrity
             'misc/*.dat',
             'misc/*.dat.gz',
             'misc/user/*png',
+            'misc/package',
             'misc/package/WebAppGallery/*.xml',
             'misc/package/WebAppGallery/install.sql',
             'vendor/autoload.php',
             'vendor/composer/autoload_real.php',
             'tmp/*',
+            // Files below are not expected but they used to be present in older Piwik versions and may be still here
+            // As they are not going to cause any trouble we won't report them as 'File to delete'
+            '*.coveralls.yml',
+            '*.scrutinizer.yml',
+            '*.gitignore',
+            '*.gitkeep',
+            '*.gitmodules',
+            '*.gitattributes',
+            '*.bower.json',
+            '*.travis.yml',
         );
     }
 
+    protected static function getMessagesDirectoriesFoundButNotExpected($messages)
+    {
+        $directoriesFoundButNotExpected = self::getDirectoriesFoundButNotExpected();
+        if (count($directoriesFoundButNotExpected) > 0) {
+
+            $messageDirectoriesToDelete = '';
+            foreach ($directoriesFoundButNotExpected as $directoryFoundNotExpected) {
+                $messageDirectoriesToDelete .= Piwik::translate('General_ExceptionDirectoryToDelete', $directoryFoundNotExpected) . '<br/>';
+            }
+
+
+            foreach ($directoriesFoundButNotExpected as $directoryFoundNotExpected) {
+                $directories[] = realpath($directoryFoundNotExpected);
+            }
+            $deleteAllAtOnce = sprintf('rm -Rf %s', implode(' ', $directories));
+
+            $messages[] = Piwik::translate('General_ExceptionUnexpectedDirectory')
+                . '<br/>'
+                . '--> ' . Piwik::translate('General_ExceptionUnexpectedDirectoryPleaseDelete') . ' <--'
+                . '<br/><br/>'
+                . $messageDirectoriesToDelete
+                . '<br/><br/>'
+                . Piwik::translate('General_ToDeleteAllDirectoriesRunThisCommand')
+                . '<br/>'
+                . $deleteAllAtOnce
+                . '<br/><br/>';
+
+        }
+
+        return $messages;
+    }
 
     /**
      * @param $messages
@@ -88,12 +135,23 @@ class FileIntegrity
             foreach ($filesFoundButNotExpected as $fileFoundNotExpected) {
                 $messageFilesToDelete .= Piwik::translate('General_ExceptionFileToDelete', $fileFoundNotExpected) . '<br/>';
             }
+
+            foreach ($filesFoundButNotExpected as $fileFoundNotExpected) {
+                $files[] = '"' . realpath($fileFoundNotExpected) . '"';
+            }
+            $deleteAllAtOnce = sprintf('rm %s', implode(' ', $files));
+
             $messages[] = Piwik::translate('General_ExceptionUnexpectedFile')
                 . '<br/>'
                 . '--> ' . Piwik::translate('General_ExceptionUnexpectedFilePleaseDelete') . ' <--'
                 . '<br/><br/>'
                 . $messageFilesToDelete
-                . '<br/>';
+                . '<br/><br/>'
+                . Piwik::translate('General_ToDeleteAllFilesRunThisCommand')
+                . '<br/>'
+                . $deleteAllAtOnce
+                . '<br/><br/>';
+
             return $messages;
 
         }
@@ -101,6 +159,45 @@ class FileIntegrity
     }
 
     /**
+     * Look for whole directories which are in the filesystem, but should not be
+     *
+     * @return array
+     */
+    protected static function getDirectoriesFoundButNotExpected()
+    {
+        static $cache = null;
+        if(!is_null($cache)) {
+            return $cache;
+        }
+
+        $pluginsInManifest = self::getPluginsFoundInManifest();
+        $directoriesInManifest = self::getDirectoriesFoundInManifest();
+        $directoriesFoundButNotExpected = array();
+
+        foreach (self::getPathsToInvestigate() as $file) {
+            $file = substr($file, 2); // remove starting characters ./ to match format in manifest.inc.php
+            $directory = dirname($file);
+
+            if(in_array($directory, $directoriesInManifest)) {
+                continue;
+            }
+
+            if (self::isFileNotInManifestButExpectedAnyway($file)) {
+                continue;
+            }
+            if (self::isFileFromPluginNotInManifest($file, $pluginsInManifest)) {
+                continue;
+            }
+
+            if (!in_array($directory, $directoriesFoundButNotExpected)) {
+                $directoriesFoundButNotExpected[] = $directory;
+            }
+        }
+
+        $cache = self::getParentDirectoriesFromListOfDirectories($directoriesFoundButNotExpected);
+        return $cache;
+    }
+    /**
      * Look for files which are in the filesystem, but should not be
      *
      * @return array
@@ -112,13 +209,7 @@ class FileIntegrity
 
         $filesFoundButNotExpected = array();
 
-        $filesToInvestigate = array_merge(
-            // all normal files
-            Filesystem::globr('.', '*'),
-            // all hidden files
-            Filesystem::globr('.', '.*')
-        );
-        foreach ($filesToInvestigate as $file) {
+        foreach (self::getPathsToInvestigate() as $file) {
             if (is_dir($file)) {
                 continue;
             }
@@ -130,6 +221,10 @@ class FileIntegrity
             if (self::isFileNotInManifestButExpectedAnyway($file)) {
                 continue;
             }
+            if (self::isFileFromDirectoryThatShouldBeDeleted($file)) {
+                // we already report the directory as "Directory to delete" so no need to repeat the instruction for each file
+                continue;
+            }
 
             if (!isset($files[$file])) {
                 $filesFoundButNotExpected[] = $file;
@@ -140,6 +235,35 @@ class FileIntegrity
     }
 
 
+    protected static function isFileFromDirectoryThatShouldBeDeleted($file)
+    {
+        $directoriesWillBeDeleted = self::getDirectoriesFoundButNotExpected();
+        foreach($directoriesWillBeDeleted as $directoryWillBeDeleted) {
+            if(strpos($file, $directoryWillBeDeleted) === 0) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    protected static function getDirectoriesFoundInManifest()
+    {
+        $files = \Piwik\Manifest::$files;
+
+        $directories = array();
+        foreach($files as $file => $manifestIntegrityInfo) {
+            $directory = $file;
+
+            // add this directory and each parent directory
+            while( ($directory = dirname($directory)) && $directory != '.' ) {
+                $directories[] = $directory;
+            }
+        }
+        $directories = array_unique($directories);
+        return $directories;
+
+    }
+
     protected static function getPluginsFoundInManifest()
     {
         $files = \Piwik\Manifest::$files;
@@ -237,7 +361,7 @@ class FileIntegrity
 
         if (!empty($messagesMismatch)) {
             $messages[] = Piwik::translate('General_FileIntegrityWarningReupload');
-            $messages[] = Piwik::translate('General_FileIntegrityWarningReuploadBis') . '<br/>';
+            $messages[] = '--> ' . Piwik::translate('General_FileIntegrityWarningReuploadBis') . ' <--<br/>';
             $messages = array_merge($messages, $messagesMismatch);
         }
 
@@ -285,4 +409,60 @@ class FileIntegrity
         return $pluginName;
     }
 
+    /**
+     * @return array
+     */
+    protected static function getPathsToInvestigate()
+    {
+        $filesToInvestigate = array_merge(
+        // all normal files
+            Filesystem::globr('.', '*'),
+            // all hidden files
+            Filesystem::globr('.', '.*')
+        );
+        return $filesToInvestigate;
+    }
+
+    /**
+     * @param $directoriesFoundButNotExpected
+     * @return array
+     */
+    protected static function getParentDirectoriesFromListOfDirectories($directoriesFoundButNotExpected)
+    {
+        sort($directoriesFoundButNotExpected);
+
+        $parentDirectoriesOnly = array();
+        foreach ($directoriesFoundButNotExpected as $directory) {
+            $directoryParent = self::getDirectoryParentFromList($directory, $directoriesFoundButNotExpected);
+            if($directoryParent) {
+                $parentDirectoriesOnly[] = $directoryParent;
+            }
+        }
+        $parentDirectoriesOnly = array_unique($parentDirectoriesOnly);
+
+        return $parentDirectoriesOnly;
+    }
+
+    /**
+     * When the parent directory of $directory is found within $directories, return it.
+     *
+     * @param $directory
+     * @param $directories
+     * @return string
+     */
+    protected static function getDirectoryParentFromList($directory, $directories)
+    {
+        foreach($directories as $directoryMaybeParent) {
+            if ($directory == $directoryMaybeParent) {
+                continue;
+            }
+
+            $isParentDirectory = strpos($directory, $directoryMaybeParent) === 0;
+            if ($isParentDirectory) {
+                return $directoryMaybeParent;
+            }
+        }
+        return null;
+    }
+
 }
\ No newline at end of file
diff --git a/core/Version.php b/core/Version.php
index 4c3cef867f..a1ef6e4d87 100644
--- a/core/Version.php
+++ b/core/Version.php
@@ -20,7 +20,7 @@ final class Version
      * The current Piwik version.
      * @var string
      */
-    const VERSION = '3.0.1-b1';
+    const VERSION = '3.0.1-b2';
 
     public function isStableVersion($version)
     {
diff --git a/lang/en.json b/lang/en.json
index e17ba19b0a..b50080ec4c 100644
--- a/lang/en.json
+++ b/lang/en.json
@@ -163,7 +163,10 @@
         "ExceptionMissingFile": "Missing file: %s",
         "ExceptionUnexpectedFile": "Files were found in your Piwik but we didn't expect them.",
         "ExceptionUnexpectedFilePleaseDelete": "Please delete these files to prevent errors.",
+        "ExceptionUnexpectedDirectory": "Directories were found in your Piwik but we didn't expect them.",
+        "ExceptionUnexpectedDirectoryPleaseDelete": "Please delete these directories to prevent errors.",
         "ExceptionFileToDelete": "File to delete: %s",
+        "ExceptionDirectoryToDelete": "Directory to delete: %s",
         "ExceptionNonceMismatch": "Could not verify the security token on this form.",
         "ExceptionPrivilege": "You can't access this resource as it requires a %s access.",
         "ExceptionPrivilegeAccessWebsite": "You can't access this resource as it requires an %s access for the website id = %d.",
@@ -367,6 +370,8 @@
         "TimeAgo": "%s ago",
         "TimeFormat": "Time format",
         "TimeOnPage": "Time on page",
+        "ToDeleteAllDirectoriesRunThisCommand": "To delete all these directories at once, you can run this command:",
+        "ToDeleteAllFilesRunThisCommand": "To delete all these files at once, you can run this command:",
         "Total": "Total",
         "TotalRatioTooltip": "This is %1$s of all %2$s %3$s.",
         "TotalRevenue": "Total Revenue",
diff --git a/plugins/Bandwidth b/plugins/Bandwidth
-Subproject b3db9b1d8d16704fc74afa6c5479c5ca2a8d160
+Subproject fe8483fd66e7a6ba9af0d8665629380a7f6775f
diff --git a/plugins/CustomAlerts b/plugins/CustomAlerts
-Subproject 8c861c9084a25dc36e1176496598a175a1619ee
+Subproject 8b0b11a887063354ab67e3366733ed5c19ab634
diff --git a/plugins/CustomDimensions b/plugins/CustomDimensions
-Subproject 8372a86d1681ebd98ae682d0ab0884146d1c7e9
+Subproject d74be369cc456fe9a7956dbb1e742588738d415
diff --git a/plugins/LogViewer b/plugins/LogViewer
-Subproject a9a0e5ac1fd22a2eaf981f49e882de28479536d
+Subproject e02c1d9f4f6bc00221d18313932b42e73345bab
diff --git a/plugins/QueuedTracking b/plugins/QueuedTracking
-Subproject 0547d18cbfa6497ea082321aa7498191ecb3479
+Subproject c4905a42b008c3eb0d58f8c24e541625d482ef4
diff --git a/plugins/SecurityInfo b/plugins/SecurityInfo
-Subproject aef66a071ca376813d56585974120e95a1478e3
+Subproject c620723f0ad4e94966d93df834cf07b5b0ad76b
diff --git a/plugins/TasksTimetable b/plugins/TasksTimetable
-Subproject ba305a46f9ca2b02fc6a0ffca141d2be2c3ee5a
+Subproject 18cc00ffc21916ebcd70e92b1b3416eecdab8b3
diff --git a/plugins/UserCountryMap/Controller.php b/plugins/UserCountryMap/Controller.php
index cb3b15f639..7f291a8b15 100644
--- a/plugins/UserCountryMap/Controller.php
+++ b/plugins/UserCountryMap/Controller.php
@@ -189,7 +189,7 @@ class Controller extends \Piwik\Plugin\Controller
             'hours'            => $this->translator->translate('Intl_Hours'),
             'hours_ago'        => $this->translator->translate('UserCountryMap_HoursAgo'),
             'days_ago'         => $this->translator->translate('UserCountryMap_DaysAgo'),
-            'actions'          => $this->translator->translate('VisitsSummary_NbPageviewsDescription'),
+            'actions'          => $this->translator->translate('Transitions_NumPageviews'),
             'searches'         => $this->translator->translate('UserCountryMap_Searches'),
             'goal_conversions' => $this->translator->translate('UserCountryMap_GoalConversions'),
         );
diff --git a/plugins/VisitorGenerator b/plugins/VisitorGenerator
-Subproject 3e49303e43f59027f0a7fd7fcfb5ca9060902a5
+Subproject 3cc7198c9ae509a22f020aef68071170aa3c9a6
diff --git a/tests/PHPUnit/Unit/DataTable/Filter/ColumnDeleteTest.php b/tests/PHPUnit/Unit/DataTable/Filter/ColumnDeleteTest.php
index 79fe1619cc..773482d603 100644
--- a/tests/PHPUnit/Unit/DataTable/Filter/ColumnDeleteTest.php
+++ b/tests/PHPUnit/Unit/DataTable/Filter/ColumnDeleteTest.php
@@ -35,14 +35,14 @@ class ColumnDeleteTest extends \PHPUnit_Framework_TestCase
     protected function makeDataTable_withoutVisitsColumn($appendRowWithSubtable = true)
     {
         $table = new DataTable();
-        $table->addRowFromArray(array(Row::COLUMNS => array('label' => 'row1', 'arrayColumn' => array('columnWithin' => 10))));
-        $table->addRowFromArray(array(Row::COLUMNS => array('label' => 'row2', 'arrayColumn' => array('columnWithin' => 11))));
-        $table->addRowFromArray(array(Row::COLUMNS => array('label' => 'row3', 'arrayColumn' => array('columnWithin' => 12))));
+        $table->addRowFromArray(array(Row::COLUMNS => array('label' => 'row1', 'arrayColumn' => array('visits' => 0, 'columnWithin' => 10))));
+        $table->addRowFromArray(array(Row::COLUMNS => array('label' => 'row2', 'arrayColumn' => array('visits' => 1, 'columnWithin' => 11))));
+        $table->addRowFromArray(array(Row::COLUMNS => array('label' => 'row3', 'arrayColumn' => array('visits' => 2, 'columnWithin' => 12))));
 
         if($appendRowWithSubtable) {
             $subTable = $this->makeDataTable_withoutVisitsColumn( $appendRowWithSubtable = false );
             $table->addRowFromArray(array(
-                Row::COLUMNS => array('label' => 'row4', 'arrayColumn' => array('columnWithin' => 13)),
+                Row::COLUMNS => array('label' => 'row4', 'arrayColumn' => array('visits' => 3, 'columnWithin' => 13)),
                 Row::DATATABLE_ASSOCIATED => $subTable
             ));
         }
@@ -140,9 +140,9 @@ class ColumnDeleteTest extends \PHPUnit_Framework_TestCase
     protected function makeArray_withoutVisitsColumns()
     {
         $array = array(
-            array('label' => 'row1', 'arrayColumn' => array( 'columnWithin' => 10)),
-            array('label' => 'row2', 'arrayColumn' => array( 'columnWithin' => 11)),
-            array('label' => 'row3', 'arrayColumn' => array( 'columnWithin' => 12)),
+            array('label' => 'row1', 'arrayColumn' => array('visits' => 0, 'columnWithin' => 10)),
+            array('label' => 'row2', 'arrayColumn' => array('visits' => 1, 'columnWithin' => 11)),
+            array('label' => 'row3', 'arrayColumn' => array('visits' => 2, 'columnWithin' => 12)),
         );
         return $array;
     }
diff --git a/tests/UI/expected-screenshots/Overlay_page_new_links.png b/tests/UI/expected-screenshots/Overlay_page_new_links.png
index 3cd92a27b2..40c45e79c9 100644
--- a/tests/UI/expected-screenshots/Overlay_page_new_links.png
+++ b/tests/UI/expected-screenshots/Overlay_page_new_links.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b1817a1d595d787ae8502be18eb99e1cdf85c5b2ed95eb4eeaf2b3b9446c5019
-size 95316
+oid sha256:37d5585ce3c48cb49c152a1f7b67dff826e5bad59ce2b0c07298672c99d95b70
+size 105869
diff --git a/tests/UI/expected-screenshots/PivotByDimension_pivoted.png b/tests/UI/expected-screenshots/PivotByDimension_pivoted.png
index ff1a1d1ede..17936d22f6 100644
--- a/tests/UI/expected-screenshots/PivotByDimension_pivoted.png
+++ b/tests/UI/expected-screenshots/PivotByDimension_pivoted.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:27b0bb8135ee54541672b136ed534f979f458cb1aa4191a69d4f2d2ae2fc6313
-size 69880
+oid sha256:2a937ec46b86cdd76237fb16c4b77da09ccac5016527ae5dc3e2b59a5da66d68
+size 63446
diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_fatal_error_safemode.png b/tests/UI/expected-screenshots/UIIntegrationTest_fatal_error_safemode.png
index f0d8e62af8..f21577214b 100644
--- a/tests/UI/expected-screenshots/UIIntegrationTest_fatal_error_safemode.png
+++ b/tests/UI/expected-screenshots/UIIntegrationTest_fatal_error_safemode.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5eaade9619d4cbad93e0731456fdcbb46b3d3c17494d1bfc718b7e8e2ce36256
-size 193496
+oid sha256:bbf49718ef99a89e8fce923f0fba79ccb28bca3505c852b2a6f5c12da7504489
+size 193525
diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_feedback_form.png b/tests/UI/expected-screenshots/UIIntegrationTest_feedback_form.png
index 663f19a7ba..5cf89d52cf 100644
--- a/tests/UI/expected-screenshots/UIIntegrationTest_feedback_form.png
+++ b/tests/UI/expected-screenshots/UIIntegrationTest_feedback_form.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:345a5f372e41dbb87adba671565624fcfc9efdbff12d4730a1d25ae870af5dab
-size 172545
+oid sha256:aa6a3ce329948c714a4491c2ffa497ae40884beba4ae2f2254caf796c32f1aab
+size 173927
diff --git a/tests/UI/expected-screenshots/UIIntegrationTest_visitors_realtime_map.png b/tests/UI/expected-screenshots/UIIntegrationTest_visitors_realtime_map.png
index 8090ecf532..ff42467040 100644
--- a/tests/UI/expected-screenshots/UIIntegrationTest_visitors_realtime_map.png
+++ b/tests/UI/expected-screenshots/UIIntegrationTest_visitors_realtime_map.png
@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ec9c25bddfe98899302a3a90731486c95473e23314df1ba8aa15502dd6077b77
-size 103297
+oid sha256:eb3ef27a3e3ae39417687002c9af7600824c75fdd51be475f6716319d03860fe
+size 113419
diff --git a/tests/UI/specs/Overlay_spec.js b/tests/UI/specs/Overlay_spec.js
index a0a7eb38dc..53888b14ac 100644
--- a/tests/UI/specs/Overlay_spec.js
+++ b/tests/UI/specs/Overlay_spec.js
@@ -69,7 +69,7 @@ describe("Overlay", function () {
     });
 
     it("should show stats for new links when dropdown opened", function (done) {
-        expect.screenshot("page_new_links").to.be.similar(0.002).to.be.capture(function (page) {
+        expect.screenshot("page_new_links").to.be.capture(function (page) {
             var pos = page.webpage.evaluate(function () {
                 var iframe = $('iframe'),
                     innerOffset = $('.dropdown-toggle', iframe.contents()).offset();
diff --git a/tests/UI/specs/PivotByDimension_spec.js b/tests/UI/specs/PivotByDimension_spec.js
index cfe367438b..44f6fb0c74 100644
--- a/tests/UI/specs/PivotByDimension_spec.js
+++ b/tests/UI/specs/PivotByDimension_spec.js
@@ -25,8 +25,10 @@ describe("PivotByDimension", function () {
             page.load(eventsUrl);
             page.click('.dimension:contains(Event Names)');
             showDataTableFooter(page);
-            page.click('.dropdownConfigureIcon');
-            page.click('.dataTablePivotBySubtable');
+            page.evaluate(function(){
+                $('.dropdownConfigureIcon').click();
+                $('.dataTablePivotBySubtable').click();
+            }, 2000);
             page.mouseMove({x: -15, y: -15}); // make sure nothing is highlighted
         }, done);
     });
diff --git a/tests/UI/specs/UIIntegration_spec.js b/tests/UI/specs/UIIntegration_spec.js
index b3e51288af..2519d08f3c 100644
--- a/tests/UI/specs/UIIntegration_spec.js
+++ b/tests/UI/specs/UIIntegration_spec.js
@@ -150,6 +150,10 @@ describe("UIIntegrationTest", function () { // TODO: Rename to Piwik?
             page.load("?" + urlBase + "#?" + idSite2Params + "&category=General_Visitors&subcategory=UserCountryMap_RealTimeMap"
                     + "&showDateTime=0&realtimeWindow=last2&changeVisitAlpha=0&enableAnimation=0&doNotRefreshVisits=1"
                     + "&removeOldVisits=0");
+            page.mouseMove('circle');
+            page.evaluate(function(){
+                $('.ui-tooltip:visible .rel-time').data('actiontime', Math.floor(new Date((new Date()).getTime()-(4*3600*24000))/1000));
+            });
         }, done);
     });