Merge branch 'master' into processed_metrics_metadata

Conflicts:
	tests/PHPUnit/System/expected/test_TwoVisitors_twoWebsites_differentDays_scheduled_report_in_pdf_tables_only__ScheduledReports.generateReport_month.original.pdf
	tests/PHPUnit/System/expected/test_csvExport__Live.getLastVisitsDetails_day.csv
	tests/PHPUnit/System/expected/test_ecommerceOrderWithItems_scheduled_report_in_pdf_tables_only__ScheduledReports.generateReport_week.original.pdf

6 files changed, 288 insertions, 12 deletions

diff --git a/core/Common.php b/core/Common.php
index 9bb4108739..b5ed581d82 100644
--- a/core/Common.php
+++ b/core/Common.php
@@ -471,7 +471,13 @@ class Common
             $ok = false;
 
             if ($varType === 'string') {
-                if (is_string($value)) $ok = true;
+                if (is_string($value) || is_int($value)) {
+                    $ok = true;
+                } else if (is_float($value)) {
+                    $value = Common::forceDotAsSeparatorForDecimalPoint($value);
+                    $ok    = true;
+                }
+
             } elseif ($varType === 'integer') {
                 if ($value == (string)(int)$value) $ok = true;
             } elseif ($varType === 'float') {
diff --git a/core/Piwik.php b/core/Piwik.php
index 81451af88e..bc5244dd52 100644
--- a/core/Piwik.php
+++ b/core/Piwik.php
@@ -481,6 +481,7 @@ class Piwik
      * in case another Login plugin is being used.
      *
      * @return string
+     * @api
      */
     public static function getLoginPluginName()
     {
diff --git a/core/Updates/2.0.3-b7.php b/core/Updates/2.0.3-b7.php
index add119e462..9903d999c1 100644
--- a/core/Updates/2.0.3-b7.php
+++ b/core/Updates/2.0.3-b7.php
@@ -23,9 +23,11 @@ class Updates_2_0_3_b7 extends Updates
         $errors = array();
 
         try {
+            $checker = new DoNotTrackHeaderChecker();
+
             // enable DoNotTrack check in PrivacyManager if DoNotTrack plugin was enabled
             if (\Piwik\Plugin\Manager::getInstance()->isPluginActivated('DoNotTrack')) {
-                DoNotTrackHeaderChecker::activate();
+                $checker->activate();
             }
 
             // enable IP anonymization if AnonymizeIP plugin was enabled
diff --git a/core/Updates/2.10.0-b1.php b/core/Updates/2.10.0-b1.php
new file mode 100644
index 0000000000..6a739b6468
--- /dev/null
+++ b/core/Updates/2.10.0-b1.php
@@ -0,0 +1,252 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+
+namespace Piwik\Updates;
+
+use Piwik\Common;
+use Piwik\DataAccess\ArchiveTableCreator;
+use Piwik\DataTable;
+use Piwik\Db;
+use Piwik\Updater;
+use Piwik\Updates;
+use DeviceDetector\Parser\Client\Browser AS BrowserParser;
+use Piwik\Plugins\Dashboard\Model AS DashboardModel;
+
+/**
+ * This Update script will update all browser and os archives of UserSettings and DevicesDetection plugin
+ *
+ * In the future only DevicesDetection will handle browser and os archives, so we try to rename all existing archives
+ * of UserSettings plugin to their corresponding archive name in DevicesDetection plugin:
+ * - *UserSettings_browser* will now be *DevicesDetection_browserVersions*
+ * - *UserSettings_os* will now be *DevicesDetection_osVersions*
+ *
+ * Unlike DevicesDetection plugin, the UserSettings plugin did not store archives holding the os and browser data without
+ * their version number. The "version-less" reports were always generated out of the "version-containing" archives .
+ * For big archives (month/year) that ment that some of the data was truncated, due to the datatable entry limit.
+ * To avoid that data loss / inaccuracy in the future, DevicesDetection plugin will also store archives without the version.
+ * For data archived after DevicesDetection plugin was enabled, those archive already exist. As we are removing the
+ * UserSettings reports, we need to move the existing old data to the new archives, which means we need to build up
+ * those archives, where they do not exist.
+ *
+ * NOTE: Some archives might not contain "all" data.
+ * That might have happened directly after the day DevicesDetection plugin was enabled. For the days before, there were
+ * no archives calculated. So week/month/year archives will only contain data for the days, where archives were generated
+ * To find a date after which it is safe to use DevicesDetection archives we need to find the first day-archive that
+ * contains DevicesDetection data. Day archives will always contain full data, but week/month/year archives may not.
+ * So we need to recreate those week/month/year archives.
+ */
+class Updates_2_10_0_b1 extends Updates
+{
+
+    static function getSql()
+    {
+        $sqls = array('# ATTENTION: This update script will execute some more SQL queries than that below as it is necessary to rebuilt some archives #' => false);
+
+        // update scheduled reports to use new plugin
+        $reportsToReplace = array(
+            'UserSettings_getBrowserVersion' => 'DevicesDetection_getBrowserVersions',
+            'UserSettings_getBrowser' => 'DevicesDetection_getBrowsers',
+            'UserSettings_getOSFamily' => 'DevicesDetection_getOsFamilies',
+            'UserSettings_getOS' => 'DevicesDetection_getOsVersions',
+            'UserSettings_getMobileVsDesktop' => 'DevicesDetection_getType',
+            'UserSettings_getBrowserType' => 'DevicesDetection_getBrowserEngines',
+            'UserSettings_getWideScreen' => 'UserSettings_getScreenType',
+        );
+
+        foreach ($reportsToReplace as $old => $new) {
+            $sqls["UPDATE " . Common::prefixTable('report') . " SET reports = REPLACE(reports, '".$old."', '".$new."')"] = false;
+        }
+
+        // update dashboard to use new widgets
+        $oldWidgets = array(
+            array('module' => 'UserSettings', 'action' => 'getBrowserVersion', 'params' => array()),
+            array('module' => 'UserSettings', 'action' => 'getBrowser', 'params' => array()),
+            array('module' => 'UserSettings', 'action' => 'getOSFamily', 'params' => array()),
+            array('module' => 'UserSettings', 'action' => 'getOS', 'params' => array()),
+            array('module' => 'UserSettings', 'action' => 'getMobileVsDesktop', 'params' => array()),
+            array('module' => 'UserSettings', 'action' => 'getBrowserType', 'params' => array()),
+            array('module' => 'UserSettings', 'action' => 'getWideScreen', 'params' => array()),
+        );
+
+        $newWidgets = array(
+            array('module' => 'DevicesDetection', 'action' => 'getBrowserVersions', 'params' => array()),
+            array('module' => 'DevicesDetection', 'action' => 'getBrowsers', 'params' => array()),
+            array('module' => 'DevicesDetection', 'action' => 'getOsFamilies', 'params' => array()),
+            array('module' => 'DevicesDetection', 'action' => 'getOsVersions', 'params' => array()),
+            array('module' => 'DevicesDetection', 'action' => 'getType', 'params' => array()),
+            array('module' => 'DevicesDetection', 'action' => 'getBrowserEngines', 'params' => array()),
+            array('module' => 'UserSettings', 'action' => 'getScreenType', 'params' => array()),
+        );
+
+        $allDashboards = Db::get()->fetchAll(sprintf("SELECT * FROM %s", Common::prefixTable('user_dashboard')));
+
+        foreach($allDashboards AS $dashboard) {
+
+            $dashboardLayout = json_decode($dashboard['layout']);
+
+            $dashboardLayout = DashboardModel::replaceDashboardWidgets($dashboardLayout, $oldWidgets, $newWidgets);
+
+            $newLayout = json_encode($dashboardLayout);
+            if ($newLayout != $dashboard['layout']) {
+                $sqls["UPDATE " . Common::prefixTable('user_dashboard') . " SET layout = '".addslashes($newLayout)."' WHERE iddashboard = ".$dashboard['iddashboard']] = false;
+            }
+        }
+
+        return $sqls;
+    }
+
+    static function update()
+    {
+        Updater::updateDatabase(__FILE__, self::getSql());
+
+        $archiveBlobTables = self::getAllArchiveBlobTables();
+
+        foreach ($archiveBlobTables as $table) {
+            self::updateBrowserArchives($table);
+            self::updateOsArchives($table);
+        }
+    }
+
+    /**
+     * Returns all available archive blob tables
+     *
+     * @return array
+     */
+    public static function getAllArchiveBlobTables()
+    {
+        static $archiveBlobTables;
+
+        if (empty($archiveBlobTables)) {
+
+            $archiveTables = ArchiveTableCreator::getTablesArchivesInstalled();
+
+            $archiveBlobTables = array_filter($archiveTables, function($name) {
+                return ArchiveTableCreator::getTypeFromTableName($name) == ArchiveTableCreator::BLOB_TABLE;
+            });
+
+            // sort tables so we have them in order of their date
+            rsort($archiveBlobTables);
+        }
+
+        return (array) $archiveBlobTables;
+    }
+
+    /**
+     * Find the first day on which DevicesDetection archives were generated
+     *
+     * @return int    Timestamp
+     */
+    public static function getFirstDayOfArchivedDeviceDetectorData()
+    {
+        static $deviceDetectionBlobAvailableDate;
+
+        if (empty($deviceDetectionBlobAvailableDate)) {
+
+            $archiveBlobTables = self::getAllArchiveBlobTables();
+
+            $deviceDetectionBlobAvailableDate = null;
+            foreach ($archiveBlobTables as $table) {
+
+                // Look for all day archives and try to find that with the lowest date
+                $deviceDetectionBlobAvailableDate = Db::get()->fetchOne(sprintf("SELECT date1 FROM %s WHERE name = 'DevicesDetection_browserVersions' AND period = 1 ORDER BY date1 ASC LIMIT 1", $table));
+
+                if (!empty($deviceDetectionBlobAvailableDate)) {
+                    break;
+                }
+
+            }
+
+            $deviceDetectionBlobAvailableDate = strtotime($deviceDetectionBlobAvailableDate);
+        }
+
+        return $deviceDetectionBlobAvailableDate;
+    }
+
+    /**
+     * Updates all browser archives to new structure
+     * @param string $table
+     * @throws \Exception
+     */
+    public static function updateBrowserArchives($table)
+    {
+        // rename old UserSettings archives where no DeviceDetection archives exists
+        Db::exec(sprintf("UPDATE IGNORE %s SET name='DevicesDetection_browserVersions' WHERE name = 'UserSettings_browser'", $table));
+
+        /*
+         * check dates of remaining (non-day) archives with calculated safe date
+         * archives before or within that week/month/year of that date will be replaced
+         */
+        $oldBrowserBlobs = Db::get()->fetchAll(sprintf("SELECT * FROM %s WHERE name = 'UserSettings_browser' AND `period` > 1", $table));
+        foreach ($oldBrowserBlobs as $blob) {
+
+            // if start date of blob is before calculated date us old usersettings archive instead of already existing DevicesDetection archive
+            if (strtotime($blob['date1']) < self::getFirstDayOfArchivedDeviceDetectorData()) {
+
+                Db::get()->query(sprintf("DELETE FROM %s WHERE idarchive = ? AND name = ?", $table), array($blob['idarchive'], 'DevicesDetection_browserVersions'));
+                Db::get()->query(sprintf("UPDATE %s SET name = ? WHERE idarchive = ? AND name = ?", $table), array('DevicesDetection_browserVersions', $blob['idarchive'], 'UserSettings_browser'));
+            }
+        }
+
+        // rebuild archives without versions
+        $browserBlobs = Db::get()->fetchAll(sprintf("SELECT * FROM %s WHERE name = 'DevicesDetection_browserVersions'", $table));
+        foreach ($browserBlobs as $blob) {
+            self::createArchiveBlobWithoutVersions($blob, 'DevicesDetection_browsers', $table);
+        }
+    }
+
+    public static function updateOsArchives($table) {
+        Db::exec(sprintf("UPDATE IGNORE %s SET name='DevicesDetection_osVersions' WHERE name = 'UserSettings_os'", $table));
+
+        /*
+         * check dates of remaining (non-day) archives with calculated safe date
+         * archives before or within that week/month/year of that date will be replaced
+         */
+        $oldOsBlobs = Db::get()->fetchAll(sprintf("SELECT * FROM %s WHERE name = 'UserSettings_os' AND `period` > 1", $table));
+        foreach ($oldOsBlobs as $blob) {
+
+            // if start date of blob is before calculated date us old usersettings archive instead of already existing DevicesDetection archive
+            if (strtotime($blob['date1']) < self::getFirstDayOfArchivedDeviceDetectorData()) {
+
+                Db::get()->query(sprintf("DELETE FROM %s WHERE idarchive = ? AND name = ?", $table), array($blob['idarchive'], 'DevicesDetection_osVersions'));
+                Db::get()->query(sprintf("UPDATE %s SET name = ? WHERE idarchive = ? AND name = ?", $table), array('DevicesDetection_osVersions', $blob['idarchive'], 'UserSettings_os'));
+            }
+        }
+
+        // rebuild archives without versions
+        $osBlobs = Db::get()->fetchAll(sprintf("SELECT * FROM %s WHERE name = 'DevicesDetection_osVersions'", $table));
+        foreach ($osBlobs as $blob) {
+            self::createArchiveBlobWithoutVersions($blob, 'DevicesDetection_os', $table);
+        }
+    }
+
+    protected static function createArchiveBlobWithoutVersions($blob, $newName, $table)
+    {
+        $blob['value'] = @gzuncompress($blob['value']);
+
+        $datatable = DataTable::fromSerializedArray($blob['value']);
+        $datatable->filter('GroupBy', array('label', function ($label) {
+            if (preg_match("/(.+) [0-9]+(?:\.[0-9]+)?$/", $label, $matches)) {
+                return $matches[1]; // should match for browsers
+            }
+
+            if (strpos($label, ';')) {
+                return substr($label, 0, 3); // should match for os
+            }
+
+            return $label;
+        }));
+
+        $newData = $datatable->getSerialized();
+
+        $blob['value'] = @gzcompress($newData[0]);
+        $blob['name'] = $newName;
+
+        Db::get()->query(sprintf('REPLACE INTO %s (`idarchive`, `name`, `idsite`, `date1`, `date2`, `period`, `ts_archived`, `value`) VALUES (?, ? , ?, ?, ?, ?, ?, ?)', $table), array_values($blob));
+    }
+}
diff --git a/core/Version.php b/core/Version.php
index 80614277c5..fcf7e8f40d 100644
--- a/core/Version.php
+++ b/core/Version.php
@@ -20,5 +20,5 @@ final class Version
      * The current Piwik version.
      * @var string
      */
-    const VERSION = '2.9.1';
+    const VERSION = '2.10.0-b1';
 }
diff --git a/core/WidgetsList.php b/core/WidgetsList.php
index e67f6b1b77..943dfceb68 100644
--- a/core/WidgetsList.php
+++ b/core/WidgetsList.php
@@ -159,18 +159,15 @@ class WidgetsList extends Singleton
     }
 
     /**
-     * Adds a report to the list of dashboard widgets.
+     * Returns the unique id of an widget with the given parameters
      *
-     * @param string $widgetCategory The widget category. This can be a translation token.
-     * @param string $widgetName The name of the widget. This can be a translation token.
-     * @param string $controllerName The report's controller name (same as the plugin name).
-     * @param string $controllerAction The report's controller action method name.
-     * @param array $customParameters Extra query parameters that should be sent while getting
-     *                                this report.
+     * @param $controllerName
+     * @param $controllerAction
+     * @param array $customParameters
+     * @return string
      */
-    public static function add($widgetCategory, $widgetName, $controllerName, $controllerAction, $customParameters = array())
+    public static function getWidgetUniqueId($controllerName, $controllerAction, $customParameters = array())
     {
-        $widgetName     = Piwik::translate($widgetName);
         $widgetUniqueId = 'widget' . $controllerName . $controllerAction;
 
         foreach ($customParameters as $name => $value) {
@@ -182,6 +179,24 @@ class WidgetsList extends Singleton
             $widgetUniqueId .= $name . $value;
         }
 
+        return $widgetUniqueId;
+    }
+
+    /**
+     * Adds a report to the list of dashboard widgets.
+     *
+     * @param string $widgetCategory The widget category. This can be a translation token.
+     * @param string $widgetName The name of the widget. This can be a translation token.
+     * @param string $controllerName The report's controller name (same as the plugin name).
+     * @param string $controllerAction The report's controller action method name.
+     * @param array $customParameters Extra query parameters that should be sent while getting
+     *                                this report.
+     */
+    public static function add($widgetCategory, $widgetName, $controllerName, $controllerAction, $customParameters = array())
+    {
+        $widgetName     = Piwik::translate($widgetName);
+        $widgetUniqueId = self::getWidgetUniqueId($controllerName, $controllerAction, $customParameters);
+
         if (!array_key_exists($widgetCategory, self::$widgets)) {
             self::$widgets[$widgetCategory] = array();
         }