Merge branch 'master' into di-config

Conflicts:
	core/FrontController.php

16 files changed, 237 insertions, 68 deletions

diff --git a/core/Common.php b/core/Common.php
index 6dfa2d22fa..823e884af1 100644
--- a/core/Common.php
+++ b/core/Common.php
@@ -34,6 +34,7 @@ class Common
     /*
      * Database
      */
+    const LANGUAGE_CODE_INVALID = 'xx';
 
     /**
      * Hashes a string into an integer which should be very low collision risks
@@ -937,8 +938,8 @@ class Common
      */
     public static function getCountry($lang, $enableLanguageToCountryGuess, $ip)
     {
-        if (empty($lang) || strlen($lang) < 2 || $lang == 'xx') {
-            return 'xx';
+        if (empty($lang) || strlen($lang) < 2 || $lang == self::LANGUAGE_CODE_INVALID) {
+            return self::LANGUAGE_CODE_INVALID;
         }
 
         $validCountries = self::getCountriesList();
@@ -974,35 +975,73 @@ class Common
                 }
             }
         }
-        return 'xx';
+        return self::LANGUAGE_CODE_INVALID;
     }
 
     /**
-     * Returns the visitor language based only on the Browser 'accepted language' information
+     * Returns the language and region string, based only on the Browser 'accepted language' information.
+     * * The language tag is defined by ISO 639-1
      *
      * @param string $browserLanguage Browser's accepted langauge header
      * @param array $validLanguages array of valid language codes
-     * @return string  2 letter ISO 639 code
+     * @return string  2 letter ISO 639 code  'es' (Spanish)
      */
-    public static function extractLanguageCodeFromBrowserLanguage($browserLanguage, $validLanguages)
+    public static function extractLanguageCodeFromBrowserLanguage($browserLanguage, $validLanguages = array())
     {
-        // assumes language preference is sorted;
-        // does not handle language-script-region tags or language range (*)
-        if (!empty($validLanguages) && preg_match_all('/(?:^|,)([a-z]{2,3})([-][a-z]{2})?/', $browserLanguage, $matches, PREG_SET_ORDER)) {
-            foreach ($matches as $parts) {
-                if (count($parts) == 3) {
-                    // match locale (language and location)
-                    if (in_array($parts[1] . $parts[2], $validLanguages)) {
-                        return $parts[1] . $parts[2];
-                    }
+        $validLanguages = self::checkValidLanguagesIsSet($validLanguages);
+        $languageRegionCode = self::extractLanguageAndRegionCodeFromBrowserLanguage($browserLanguage, $validLanguages);
+
+        if(strlen($languageRegionCode) == 2) {
+            $languageCode = $languageRegionCode;
+        } else {
+            $languageCode = substr($languageRegionCode, 0, 2);
+        }
+        if(in_array($languageCode, $validLanguages)) {
+            return $languageCode;
+        }
+        return self::LANGUAGE_CODE_INVALID;
+    }
+
+    /**
+     * Returns the language and region string, based only on the Browser 'accepted language' information.
+     * * The language tag is defined by ISO 639-1
+     * * The region tag is defined by ISO 3166-1
+     *
+     * @param string $browserLanguage Browser's accepted langauge header
+     * @param array $validLanguages array of valid language codes. Note that if the array includes "fr" then it will consider all regional variants of this language valid, such as "fr-ca" etc.
+     * @return string 2 letter ISO 639 code 'es' (Spanish) or if found, includes the region as well: 'es-ar'
+     */
+    public static function extractLanguageAndRegionCodeFromBrowserLanguage($browserLanguage, $validLanguages = array() )
+    {
+        $validLanguages = self::checkValidLanguagesIsSet($validLanguages);
+
+        if(!preg_match_all('/(?:^|,)([a-z]{2,3})([-][a-z]{2})?/', $browserLanguage, $matches, PREG_SET_ORDER)) {
+            return self::LANGUAGE_CODE_INVALID;
+        }
+        foreach ($matches as $parts) {
+            $langIso639 = $parts[1];
+            if(empty($langIso639)) {
+                continue;
+            }
+
+            // If a region tag is found eg. "fr-ca"
+            if (count($parts) == 3) {
+                $regionIso3166 = $parts[2]; // eg. "-ca"
+
+                if (in_array($langIso639 . $regionIso3166, $validLanguages)) {
+                    return $langIso639 . $regionIso3166;
                 }
-                // match language only (where no region provided)
-                if (in_array($parts[1], $validLanguages)) {
-                    return $parts[1];
+
+                if (in_array($langIso639, $validLanguages)) {
+                    return $langIso639 . $regionIso3166;
                 }
             }
+            // eg. "fr" or "es"
+            if (in_array($langIso639, $validLanguages)) {
+                return $langIso639;
+            }
         }
-        return 'xx';
+        return self::LANGUAGE_CODE_INVALID;
     }
 
     /**
@@ -1161,4 +1200,17 @@ class Common
             }
         }
     }
+
+    /**
+     * @param $validLanguages
+     * @return array
+     */
+    protected static function checkValidLanguagesIsSet($validLanguages)
+    {
+        if (empty($validLanguages)) {
+            $validLanguages = array_keys(Common::getLanguagesList());
+            return $validLanguages;
+        }
+        return $validLanguages;
+    }
 }
diff --git a/core/DataAccess/ArchiveWriter.php b/core/DataAccess/ArchiveWriter.php
index 7e0086d795..1fcacf790d 100644
--- a/core/DataAccess/ArchiveWriter.php
+++ b/core/DataAccess/ArchiveWriter.php
@@ -51,13 +51,13 @@ class ArchiveWriter
     const DONE_INVALIDATED = 4;
 
     protected $fields = array('idarchive',
-                              'idsite',
-                              'date1',
-                              'date2',
-                              'period',
-                              'ts_archived',
-                              'name',
-                              'value');
+        'idsite',
+        'date1',
+        'date2',
+        'period',
+        'ts_archived',
+        'name',
+        'value');
 
     public function __construct(ArchiveProcessor\Parameters $params, $isArchiveTemporary)
     {
@@ -234,11 +234,11 @@ class ArchiveWriter
     protected function getInsertRecordBind()
     {
         return array($this->getIdArchive(),
-                     $this->idSite,
-                     $this->dateStart->toString('Y-m-d'),
-                     $this->period->getDateEnd()->toString('Y-m-d'),
-                     $this->period->getId(),
-                     date("Y-m-d H:i:s"));
+            $this->idSite,
+            $this->dateStart->toString('Y-m-d'),
+            $this->period->getDateEnd()->toString('Y-m-d'),
+            $this->period->getId(),
+            date("Y-m-d H:i:s"));
     }
 
     protected function getTableNameToInsert($value)
diff --git a/core/DataTable/Filter/GroupBy.php b/core/DataTable/Filter/GroupBy.php
index 9391ccd95f..2ac79a6de9 100755
--- a/core/DataTable/Filter/GroupBy.php
+++ b/core/DataTable/Filter/GroupBy.php
@@ -10,6 +10,7 @@ namespace Piwik\DataTable\Filter;
 
 use Piwik\DataTable;
 use Piwik\DataTable\BaseFilter;
+use Piwik\DataTable\Row;
 
 /**
  * DataTable filter that will group {@link DataTable} rows together based on the results
@@ -71,6 +72,7 @@ class GroupBy extends BaseFilter
      */
     public function filter($table)
     {
+        /** @var Row[] $groupByRows */
         $groupByRows = array();
         $nonGroupByRowIds = array();
 
diff --git a/core/DataTable/Row.php b/core/DataTable/Row.php
index a90855a1d9..c6c0d21e9f 100644
--- a/core/DataTable/Row.php
+++ b/core/DataTable/Row.php
@@ -640,10 +640,7 @@ class Row implements \ArrayAccess, \IteratorAggregate
             return $newValue;
         }
 
-        if (is_string($columnToSumValue)) {
-            throw new Exception("Trying to add two strings in DataTable\Row::sumRowArray: "
-                              . "'$thisColumnValue' + '$columnToSumValue'" . " for row " . $this->__toString());
-        }
+        $this->warnWhenSummingTwoStrings($thisColumnValue, $columnToSumValue);
 
         return 0;
     }
@@ -761,4 +758,16 @@ class Row implements \ArrayAccess, \IteratorAggregate
         }
     }
 
+    protected function warnWhenSummingTwoStrings($thisColumnValue, $columnToSumValue)
+    {
+        if (is_string($columnToSumValue)) {
+            Log::warning(
+                "Trying to add two strings in DataTable\Row::sumRowArray: %s + %s for row %s",
+                $thisColumnValue,
+                $columnToSumValue,
+                $this->__toString()
+            );
+        }
+    }
+
 }
diff --git a/core/Exception/AuthenticationFailedException.php b/core/Exception/AuthenticationFailedException.php
new file mode 100644
index 0000000000..ca3efc25c1
--- /dev/null
+++ b/core/Exception/AuthenticationFailedException.php
@@ -0,0 +1,13 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Exception;
+
+class AuthenticationFailedException extends Exception
+{
+}
\ No newline at end of file
diff --git a/core/Exceptions/HtmlMessageException.php b/core/Exception/Exception.php
index 243a0f1726..106034bf5a 100644
--- a/core/Exceptions/HtmlMessageException.php
+++ b/core/Exception/Exception.php
@@ -6,25 +6,25 @@
  * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
  *
  */
-namespace Piwik\Exceptions;
+namespace Piwik\Exception;
 
-use Exception;
+use Exception as PhpException;
 
 /**
  * An exception whose message has HTML content. When these exceptions are caught
  * the message will not be sanitized before being displayed to the user.
- *
- * @api
  */
-class HtmlMessageException extends Exception
+abstract class Exception extends PhpException
 {
-    /**
-     * Returns the exception message.
-     *
-     * @return string
-     */
-    public function getHtmlMessage()
+    private $isHtmlMessage = false;
+
+    public function setIsHtmlMessage()
+    {
+        $this->isHtmlMessage = true;
+    }
+
+    public function isHtmlMessage()
     {
-        return $this->getMessage();
+        return $this->isHtmlMessage;
     }
 }
\ No newline at end of file
diff --git a/core/Exception/MissingFilePermissionException.php b/core/Exception/MissingFilePermissionException.php
new file mode 100644
index 0000000000..268725c70b
--- /dev/null
+++ b/core/Exception/MissingFilePermissionException.php
@@ -0,0 +1,13 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Exception;
+
+class MissingFilePermissionException extends Exception
+{
+}
\ No newline at end of file
diff --git a/core/Exception/NoPrivilegesException.php b/core/Exception/NoPrivilegesException.php
new file mode 100644
index 0000000000..3ee8c9ce2b
--- /dev/null
+++ b/core/Exception/NoPrivilegesException.php
@@ -0,0 +1,13 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Exception;
+
+class NoPrivilegesException extends Exception
+{
+}
\ No newline at end of file
diff --git a/core/Exception/NoWebsiteFoundException.php b/core/Exception/NoWebsiteFoundException.php
new file mode 100644
index 0000000000..eb97cad37a
--- /dev/null
+++ b/core/Exception/NoWebsiteFoundException.php
@@ -0,0 +1,13 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Exception;
+
+class NoWebsiteFoundException extends Exception
+{
+}
\ No newline at end of file
diff --git a/core/Filechecks.php b/core/Filechecks.php
index eab93fd260..0c37680fc4 100644
--- a/core/Filechecks.php
+++ b/core/Filechecks.php
@@ -8,7 +8,7 @@
  */
 namespace Piwik;
 
-use Piwik\Exceptions\HtmlMessageException;
+use Piwik\Exception\MissingFilePermissionException;
 
 class Filechecks
 {
@@ -96,7 +96,7 @@ class Filechecks
             }
         }
 
-        $directoryMessage = "<p><b>Piwik couldn't write to some directories $optionalUserInfo</b>.</p>";
+        $directoryMessage  = "<p><b>Piwik couldn't write to some directories $optionalUserInfo</b>.</p>";
         $directoryMessage .= "<p>Try to Execute the following commands on your server, to allow Write access on these directories"
             . ":</p>"
             . "<blockquote>$directoryList</blockquote>"
@@ -104,7 +104,10 @@ class Filechecks
             . "<p>After applying the modifications, you can <a href='index.php'>refresh the page</a>.</p>"
             . "<p>If you need more help, try <a href='?module=Proxy&action=redirect&url=http://piwik.org'>Piwik.org</a>.</p>";
 
-        throw new HtmlMessageException($directoryMessage);
+        $ex = new MissingFilePermissionException($directoryMessage);
+        $ex->setIsHtmlMessage();
+
+        throw $ex;
     }
 
     /**
diff --git a/core/FrontController.php b/core/FrontController.php
index f49fa94655..553f6b393f 100644
--- a/core/FrontController.php
+++ b/core/FrontController.php
@@ -13,7 +13,7 @@ use Exception;
 use Piwik\API\Request;
 use Piwik\API\ResponseBuilder;
 use Piwik\Container\StaticContainer;
-use Piwik\Exceptions\HtmlMessageException;
+use Piwik\Exception\AuthenticationFailedException;
 use Piwik\Http\Router;
 use Piwik\Plugin\Controller;
 use Piwik\Plugin\Report;
@@ -421,10 +421,15 @@ class FrontController extends Singleton
         try {
             $authAdapter = Registry::get('auth');
         } catch (Exception $e) {
-            throw new HtmlMessageException("Authentication object cannot be found in the Registry. Maybe the Login plugin is not activated?
-                            <br />You can activate the plugin by adding:<br />
-                            <code>Plugins[] = Login</code><br />
-                            under the <code>[Plugins]</code> section in your config/config.ini.php");
+            $message = "Authentication object cannot be found in the Registry. Maybe the Login plugin is not activated?
+                        <br />You can activate the plugin by adding:<br />
+                        <code>Plugins[] = Login</code><br />
+                        under the <code>[Plugins]</code> section in your config/config.ini.php";
+
+            $ex = new AuthenticationFailedException($message);
+            $ex->setIsHtmlMessage();
+
+            throw $ex;
         }
         Access::getInstance()->reloadAccess($authAdapter);
 
@@ -615,10 +620,10 @@ class FrontController extends Singleton
     {
         $debugTrace = $ex->getTraceAsString();
 
-        if (method_exists($ex, 'getHtmlMessage')) {
-            $message = $ex->getHtmlMessage();
-        } else {
-            $message = Common::sanitizeInputValue($ex->getMessage());
+        $message = $ex->getMessage();
+
+        if (!method_exists($ex, 'isHtmlMessage') || !$ex->isHtmlMessage()) {
+            $message = Common::sanitizeInputValue($message);
         }
 
         $logo = new CustomLogo();
diff --git a/core/Period.php b/core/Period.php
index 1e3f188b5e..eb3c0fda89 100644
--- a/core/Period.php
+++ b/core/Period.php
@@ -263,7 +263,7 @@ abstract class Period
     abstract public function getLocalizedLongString();
 
     /**
-     * Returns a succinct string describing this period.
+     * Returns the date range string comprising two dates
      *
      * @return string eg, `'2012-01-01,2012-01-31'`.
      */
diff --git a/core/Plugin/Controller.php b/core/Plugin/Controller.php
index 9985c100d5..4afcbf25ad 100644
--- a/core/Plugin/Controller.php
+++ b/core/Plugin/Controller.php
@@ -17,7 +17,8 @@ use Piwik\Config as PiwikConfig;
 use Piwik\Config;
 use Piwik\DataTable\Filter\CalculateEvolutionFilter;
 use Piwik\Date;
-use Piwik\Exceptions\HtmlMessageException;
+use Piwik\Exception\NoPrivilegesException;
+use Piwik\Exception\NoWebsiteFoundException;
 use Piwik\FrontController;
 use Piwik\Menu\MenuTop;
 use Piwik\Menu\MenuUser;
@@ -841,16 +842,22 @@ abstract class Controller
             $message = "Error: no website was found in this Piwik installation.
 			<br />Check the table '$siteTableName' in your database, it should contain your Piwik websites.";
 
-            throw new HtmlMessageException($message);
+            $ex = new NoWebsiteFoundException($message);
+            $ex->setIsHtmlMessage();
+
+            throw $ex;
         }
 
         if (!Piwik::isUserIsAnonymous()) {
             $currentLogin = Piwik::getCurrentUserLogin();
             $emails = implode(',', Piwik::getAllSuperUserAccessEmailAddresses());
-            $errorMessage = sprintf(Piwik::translate('CoreHome_NoPrivilegesAskPiwikAdmin'), $currentLogin, "<br/><a href='mailto:" . $emails . "?subject=Access to Piwik for user $currentLogin'>", "</a>");
+            $errorMessage  = sprintf(Piwik::translate('CoreHome_NoPrivilegesAskPiwikAdmin'), $currentLogin, "<br/><a href='mailto:" . $emails . "?subject=Access to Piwik for user $currentLogin'>", "</a>");
             $errorMessage .= "<br /><br />&nbsp;&nbsp;&nbsp;<b><a href='index.php?module=" . Registry::get('auth')->getName() . "&amp;action=logout'>&rsaquo; " . Piwik::translate('General_Logout') . "</a></b><br />";
 
-            throw new HtmlMessageException($errorMessage);
+            $ex = new NoPrivilegesException($errorMessage);
+            $ex->setIsHtmlMessage();
+
+            throw $ex;
         }
 
         echo FrontController::getInstance()->dispatch(Piwik::getLoginPluginName(), false);
diff --git a/core/Session.php b/core/Session.php
index d8fa47b4d2..f67b2e4caf 100644
--- a/core/Session.php
+++ b/core/Session.php
@@ -9,7 +9,7 @@
 namespace Piwik;
 
 use Exception;
-use Piwik\Exceptions\HtmlMessageException;
+use Piwik\Exception\MissingFilePermissionException;
 use Piwik\Session\SaveHandler\DbTable;
 use Zend_Session;
 
@@ -132,7 +132,10 @@ class Session extends Zend_Session
                 $e->getMessage()
             );
 
-            throw new HtmlMessageException($message, $e->getCode(), $e);
+            $ex = new MissingFilePermissionException($message, $e->getCode(), $e);
+            $ex->setIsHtmlMessage();
+
+            throw $ex;
         }
     }
 
diff --git a/core/Tracker/TableLogAction.php b/core/Tracker/TableLogAction.php
index 4f4849a352..709936f2a5 100644
--- a/core/Tracker/TableLogAction.php
+++ b/core/Tracker/TableLogAction.php
@@ -172,7 +172,7 @@ class TableLogAction
             $valueToMatch = preg_replace('@^http[s]?://(www\.)?@i', '', $valueToMatch);
         }
 
-        $valueToMatch = Common::sanitizeInputValue(Common::unsanitizeInputValue($valueToMatch));
+        $valueToMatch = self::normaliseActionString($actionType, $valueToMatch);
 
         if ($matchType == SegmentExpression::MATCH_EQUAL
             || $matchType == SegmentExpression::MATCH_NOT_EQUAL
@@ -232,5 +232,41 @@ class TableLogAction
         }
     }
 
+    /**
+     * This function will sanitize or not if it's needed for the specified action type
+     *
+     * URLs (Page URLs, Downloads, Outlinks) are stored raw (unsanitized)
+     * while other action types are stored Sanitized
+     *
+     * @param $actionType
+     * @param $actionString
+     * @return string
+     */
+    private static function normaliseActionString($actionType, $actionString)
+    {
+        $actionString = Common::unsanitizeInputValue($actionString);
+
+        if (self::isActionTypeStoredSanitized($actionType)) {
+            return Common::sanitizeInputValue($actionString);
+        }
+        return $actionString;
+    }
+
+    /**
+     * @param $actionType
+     * @return bool
+     */
+    private static function isActionTypeStoredSanitized($actionType)
+    {
+        $actionsTypesStoredUnsanitized = array(
+            $actionType == Action::TYPE_PAGE_URL,
+            $actionType == Action::TYPE_DOWNLOAD,
+            $actionType == Action::TYPE_OUTLINK,
+        );
+
+        $isStoredUnsanitized = in_array($actionType, $actionsTypesStoredUnsanitized);
+        return !$isStoredUnsanitized;
+    }
+
 }
 
diff --git a/core/Version.php b/core/Version.php
index 6e89402dec..fa7f0f29b7 100644
--- a/core/Version.php
+++ b/core/Version.php
@@ -20,5 +20,5 @@ final class Version
      * The current Piwik version.
      * @var string
      */
-    const VERSION = '2.9.0-b8';
+    const VERSION = '2.9.0';
 }