Merge master into "3.0"

Conflicts:
	.travis.yml
	CHANGELOG.md
	core/Plugin/ControllerAdmin.php
	core/Version.php
	core/testMinimumPhpVersion.php
	plugins/API/API.php
	plugins/Actions/Reports/GetPageUrls.php
	plugins/AnonymousPiwikUsageMeasurement
	plugins/CoreHome/CoreHome.php
	plugins/CoreHome/templates/_menu.twig
	plugins/CoreVisualizations/javascripts/jqplotEvolutionGraph.js
	plugins/CustomVariables/tests/System/expected/test_CustomVariablesSystemTest__CustomVariables.getCustomVariables_day.xml
	plugins/Goals/Controller.php
	tests/PHPUnit/Fixtures/ManyVisitsWithGeoIP.php
	tests/PHPUnit/Integration/Plugin/ManagerTest.php
	tests/PHPUnit/System/expected/test_FlattenReports__CustomVariables.getCustomVariables_day.xml
	tests/PHPUnit/System/expected/test_TwoVisitors_twoWebsites_differentDays_scheduled_report_in_html_tables_only__ScheduledReports.generateReport_month.original.html
	tests/PHPUnit/System/expected/test_apiGetReportMetadata__API.getSegmentsMetadata.xml
	tests/PHPUnit/System/expected/test_ecommerceOrderWithItems_scheduled_report_in_html_tables_only__ScheduledReports.generateReport_week.original.html
	tests/PHPUnit/System/expected/test_periodIsRange_dateIsLastN_MetadataAndNormalAPI__CustomVariables.getCustomVariables_range.xml
	tests/PHPUnit/System/expected/test_periodIsRange_dateIsLastN_MetadataAndNormalAPI_pagesegment__CustomVariables.getCustomVariables_range.xml
	tests/PHPUnit/System/expected/test_reportLimiting__CustomVariables.getCustomVariables_day.xml
	tests/PHPUnit/System/expected/test_reportLimiting_rankingQuery__CustomVariables.getCustomVariables_day.xml
	tests/PHPUnit/System/expected/test_twoVisitsWithCustomVariables_segmentMatchALL_noGoalData__CustomVariables.getCustomVariables_day.xml
	tests/PHPUnit/System/expected/test_twoVisitsWithCustomVariables_segmentMatchALL_noGoalData__CustomVariables.getCustomVariables_week.xml
	tests/PHPUnit/System/expected/test_twoVisitsWithCustomVariables_segmentMatchVisitorType__CustomVariables.getCustomVariables_day.xml
	tests/PHPUnit/System/expected/test_twoVisitsWithCustomVariables_segmentMatchVisitorType__CustomVariables.getCustomVariables_week.xml
	tests/UI/expected-ui-screenshots
	tests/UI/specs/Overlay_spec.js
	tests/UI/specs/UIIntegration_spec.js

21 files changed, 1349 insertions, 443 deletions

diff --git a/plugins/Referrers/API.php b/plugins/Referrers/API.php
index d1c08ac20c..befd651fbe 100644
--- a/plugins/Referrers/API.php
+++ b/plugins/Referrers/API.php
@@ -230,14 +230,14 @@ class API extends \Piwik\Plugin\API
         $dataTable = Archive::createDataTableFromArchive(Archiver::SEARCH_ENGINES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded, $flat);
 
         if ($flat) {
-            $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'url', __NAMESPACE__ . '\getSearchEngineUrlFromName'));
-            $dataTable->filter('MetadataCallbackAddMetadata', array('url', 'logo', __NAMESPACE__ . '\getSearchEngineLogoFromUrl'));
+            $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'url', function ($url) { return SearchEngine::getInstance()->getUrlFromName($url); }));
+            $dataTable->filter('MetadataCallbackAddMetadata', array('url', 'logo', function ($url) { return SearchEngine::getInstance()->getLogoFromUrl($url); }));
             $dataTable->filterSubtables('Piwik\Plugins\Referrers\DataTable\Filter\KeywordsFromSearchEngineId', array($dataTable));
         } else {
             $dataTable->filter('AddSegmentByLabel', array('referrerName'));
             $dataTable->queueFilter('PrependSegment', array('referrerType==search;'));
-            $dataTable->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', __NAMESPACE__ . '\getSearchEngineUrlFromName'));
-            $dataTable->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', __NAMESPACE__ . '\getSearchEngineLogoFromUrl'));
+            $dataTable->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', function ($url) { return SearchEngine::getInstance()->getUrlFromName($url); }));
+            $dataTable->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', function ($url) { return SearchEngine::getInstance()->getLogoFromUrl($url); }));
         }
 
         return $dataTable;
@@ -309,9 +309,9 @@ class API extends \Piwik\Plugin\API
     {
         $dataTable = Archive::createDataTableFromArchive(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded, false);
 
-        $dataTable->filter('ColumnCallbackDeleteRow', array('label', function ($url) { return !isSocialUrl($url); }));
-        $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'url', __NAMESPACE__ . '\getSocialMainUrl'));
-        $dataTable->filter('GroupBy', array('label', __NAMESPACE__ . '\getSocialNetworkFromDomain'));
+        $dataTable->filter('ColumnCallbackDeleteRow', array('label', function ($url) { return !Social::getInstance()->isSocialUrl($url); }));
+        $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'url', function ($url) { return Social::getInstance()->getMainUrl($url); }));
+        $dataTable->filter('GroupBy', array('label', function ($url) { return Social::getInstance()->getSocialNetworkFromDomain($url); }));
 
         $this->setSocialIdSubtables($dataTable);
         $this->removeSubtableMetadata($dataTable);
@@ -320,7 +320,7 @@ class API extends \Piwik\Plugin\API
             $this->buildExpandedTableForFlattenGetSocials($idSite, $period, $date, $segment, $expanded, $dataTable);
         }
 
-        $dataTable->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', __NAMESPACE__ . '\getSocialsLogoFromUrl'));
+        $dataTable->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', function ($url) { return Social::getInstance()->getLogoFromUrl($url); }));
 
         return $dataTable;
     }
@@ -334,7 +334,7 @@ class API extends \Piwik\Plugin\API
      * @param string $date
      * @param bool|string $segment
      * @param bool|int $idSubtable This ID does not reference a real DataTable record. Instead, it
-     *                              is the array index of an item in the /core/DataFiles/Socials.php file.
+     *                              is the array index of an item in the Socials list file.
      *                              The urls are filtered by the social network at this index.
      *                              If false, no filtering is done and every social URL is returned.
      * @return DataTable
@@ -344,7 +344,7 @@ class API extends \Piwik\Plugin\API
         $dataTable = $this->getDataTable(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded = true);
 
         // get the social network domain referred to by $idSubtable
-        $socialNetworks = Common::getSocialUrls();
+        $socialNetworks = Social::getInstance()->getDefinitions();
 
         $social = false;
         if ($idSubtable !== false) {
@@ -362,7 +362,7 @@ class API extends \Piwik\Plugin\API
         $dataTable->filter(
             'ColumnCallbackDeleteRow',
             array('label',
-                  function ($url) use ($social) { return !isSocialUrl($url, $social); }
+                  function ($url) use ($social) { return !Social::getInstance()->isSocialUrl($url, $social); }
             )
         );
 
@@ -428,7 +428,7 @@ class API extends \Piwik\Plugin\API
     /**
      * Sets the subtable IDs for the DataTable returned by getSocial.
      *
-     * The IDs are int indexes into the array in /core/DataFiles/Socials.php.
+     * The IDs are int indexes into the array in of defined socials.
      *
      * @param DataTable $dataTable
      */
@@ -443,7 +443,7 @@ class API extends \Piwik\Plugin\API
                 $socialName = $row->getColumn('label');
 
                 $i = 1; // start at one because idSubtable=0 is equivalent to idSubtable=false
-                foreach (Common::getSocialUrls() as $name) {
+                foreach (Social::getInstance()->getDefinitions() as $name) {
                     if ($name == $socialName) {
                         $row->setNonLoadedSubtableId($i);
                         break;
@@ -491,7 +491,7 @@ class API extends \Piwik\Plugin\API
     {
         $urlsTable = Archive::createDataTableFromArchive(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded, $flat = true);
         $urlsTable->filter('ColumnCallbackDeleteRow', array('label', function ($url) {
-            return !isSocialUrl($url);
+            return !Social::getInstance()->isSocialUrl($url);
         }));
         $urlsTable = $urlsTable->mergeSubtables();
 
@@ -504,7 +504,7 @@ class API extends \Piwik\Plugin\API
             $rows = $urlsTable->getRows();
             foreach ($rows as $id => $urlsTableRow) {
                 $url = $urlsTableRow->getColumn('label');
-                if (isSocialUrl($url, $social)) {
+                if (Social::getInstance()->isSocialUrl($url, $social)) {
                     $newTable->addRow($urlsTableRow);
                     $urlsTable->deleteRow($id);
                 }
diff --git a/plugins/Referrers/Columns/Base.php b/plugins/Referrers/Columns/Base.php
index 78fe27516c..a4da79d389 100644
--- a/plugins/Referrers/Columns/Base.php
+++ b/plugins/Referrers/Columns/Base.php
@@ -11,6 +11,9 @@ namespace Piwik\Plugins\Referrers\Columns;
 use Piwik\Common;
 use Piwik\Piwik;
 use Piwik\Plugin\Dimension\VisitDimension;
+use Piwik\Plugins\Referrers\SearchEngine AS SearchEngineDetection;
+use Piwik\Plugins\SitesManager\SiteUrls;
+use Piwik\Tracker\Cache;
 use Piwik\Tracker\PageUrl;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visit;
@@ -111,6 +114,14 @@ abstract class Base extends VisitDimension
         if (!$referrerDetected && !empty($this->referrerHost)) {
             $this->typeReferrerAnalyzed = Common::REFERRER_TYPE_WEBSITE;
             $this->nameReferrerAnalyzed = Common::mb_strtolower($this->referrerHost);
+
+            $urlsByHost = $this->getCachedUrlsByHostAndIdSite();
+
+            $directEntry = new SiteUrls();
+            $path = $directEntry->getPathMatchingUrl($this->referrerUrlParse, $urlsByHost);
+            if (!empty($path) && $path !== '/') {
+                $this->nameReferrerAnalyzed .= rtrim($path, '/');
+            }
         }
 
         $referrerInformation = array(
@@ -139,7 +150,7 @@ abstract class Base extends VisitDimension
      */
     protected function detectReferrerSearchEngine()
     {
-        $searchEngineInformation = UrlHelper::extractSearchEngineInformationFromUrl($this->referrerUrl);
+        $searchEngineInformation = SearchEngineDetection::getInstance()->extractInformationFromUrl($this->referrerUrl);
 
         /**
          * Triggered when detecting the search engine of a referrer URL.
@@ -241,6 +252,17 @@ abstract class Base extends VisitDimension
         return true;
     }
 
+    private function getCachedUrlsByHostAndIdSite()
+    {
+        $cache = Cache::getCacheGeneral();
+
+        if (!empty($cache['allUrlsByHostAndIdSite'])) {
+            return $cache['allUrlsByHostAndIdSite'];
+        }
+
+        return array();
+    }
+
     /**
      * We have previously tried to detect the campaign variables in the URL
      * so at this stage, if the referrer host is the current host,
@@ -250,20 +272,32 @@ abstract class Base extends VisitDimension
      */
     protected function detectReferrerDirectEntry()
     {
-        if (!empty($this->referrerHost)) {
-            // is the referrer host the current host?
-            if (isset($this->currentUrlParse['host'])) {
-                $currentHost = Common::mb_strtolower($this->currentUrlParse['host'], 'UTF-8');
-                if ($currentHost == Common::mb_strtolower($this->referrerHost, 'UTF-8')) {
-                    $this->typeReferrerAnalyzed = Common::REFERRER_TYPE_DIRECT_ENTRY;
-                    return true;
-                }
-            }
-            if (Visit::isHostKnownAliasHost($this->referrerHost, $this->idsite)) {
+        if (empty($this->referrerHost)) {
+            return false;
+        }
+
+        $urlsByHost = $this->getCachedUrlsByHostAndIdSite();
+
+        $directEntry   = new SiteUrls();
+        $matchingSites = $directEntry->getIdSitesMatchingUrl($this->referrerUrlParse, $urlsByHost);
+
+        if (isset($matchingSites) && is_array($matchingSites) && in_array($this->idsite, $matchingSites)) {
+            $this->typeReferrerAnalyzed = Common::REFERRER_TYPE_DIRECT_ENTRY;
+            return true;
+        } elseif (isset($matchingSites)) {
+            return false;
+        }
+
+        // fallback logic if the referrer domain is not known to any site to not break BC
+        if (isset($this->currentUrlParse['host'])) {
+            // this might be actually buggy if first thing tracked is eg an outlink and referrer is from that site
+            $currentHost = Common::mb_strtolower($this->currentUrlParse['host']);
+            if ($currentHost == Common::mb_strtolower($this->referrerHost)) {
                 $this->typeReferrerAnalyzed = Common::REFERRER_TYPE_DIRECT_ENTRY;
                 return true;
             }
         }
+
         return false;
     }
 
@@ -277,7 +311,7 @@ abstract class Base extends VisitDimension
 
         // Set the Campaign keyword to the keyword found in the Referrer URL if any
         if (!empty($this->nameReferrerAnalyzed)) {
-            $referrerUrlInfo = UrlHelper::extractSearchEngineInformationFromUrl($this->referrerUrl);
+            $referrerUrlInfo = SearchEngineDetection::getInstance()->extractInformationFromUrl($this->referrerUrl);
             if (!empty($referrerUrlInfo['keywords'])) {
                 $this->keywordReferrerAnalyzed = $referrerUrlInfo['keywords'];
             }
@@ -448,7 +482,7 @@ abstract class Base extends VisitDimension
 
     protected function hasReferrerColumnChanged(Visitor $visitor, $information, $infoName)
     {
-        return Common::mb_strtolower($visitor->getVisitorColumn($infoName)) != $information[$infoName];
+        return Common::mb_strtolower($visitor->getVisitorColumn($infoName)) != Common::mb_strtolower($information[$infoName]);
     }
 
     protected function doesLastActionHaveSameReferrer(Visitor $visitor, $referrerType)
diff --git a/plugins/Referrers/Columns/Campaign.php b/plugins/Referrers/Columns/Campaign.php
index ff2d5c2401..c29b622336 100644
--- a/plugins/Referrers/Columns/Campaign.php
+++ b/plugins/Referrers/Columns/Campaign.php
@@ -37,7 +37,7 @@ class Campaign extends Base
 
     /**
      * If we should create a new visit when the campaign changes, check if the campaign info changed and if so
-     * force the tracker to create a new visit.
+     * force the tracker to create a new visit.i
      *
      * @param Request $request
      * @param Visitor $visitor
diff --git a/plugins/Referrers/Columns/Keyword.php b/plugins/Referrers/Columns/Keyword.php
index 78605c9d94..b3bfc374b1 100644
--- a/plugins/Referrers/Columns/Keyword.php
+++ b/plugins/Referrers/Columns/Keyword.php
@@ -8,6 +8,7 @@
  */
 namespace Piwik\Plugins\Referrers\Columns;
 
+use Piwik\Common;
 use Piwik\Piwik;
 use Piwik\Plugins\Referrers\Segment;
 use Piwik\Tracker\Request;
@@ -44,7 +45,7 @@ class Keyword extends Base
         $information = $this->getReferrerInformationFromRequest($request);
 
         if (!empty($information['referer_keyword'])) {
-            return substr($information['referer_keyword'], 0, 255);
+            return Common::mb_substr($information['referer_keyword'], 0, 255);
         }
 
         return $information['referer_keyword'];
diff --git a/plugins/Referrers/Columns/ReferrerName.php b/plugins/Referrers/Columns/ReferrerName.php
index 7bdb0732a6..ded0a48f9e 100644
--- a/plugins/Referrers/Columns/ReferrerName.php
+++ b/plugins/Referrers/Columns/ReferrerName.php
@@ -8,6 +8,7 @@
  */
 namespace Piwik\Plugins\Referrers\Columns;
 
+use Piwik\Common;
 use Piwik\Plugins\Referrers\Segment;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visitor;
@@ -38,10 +39,8 @@ class ReferrerName extends Base
         $information = $this->getReferrerInformationFromRequest($request);
 
         if (!empty($information['referer_name'])) {
-
-            return substr($information['referer_name'], 0, 70);
+            return Common::mb_substr($information['referer_name'], 0, 70);
         }
-
         return $information['referer_name'];
     }
 
diff --git a/plugins/Referrers/DataTable/Filter/KeywordsFromSearchEngineId.php b/plugins/Referrers/DataTable/Filter/KeywordsFromSearchEngineId.php
index 1688713a86..789a1b1a6a 100644
--- a/plugins/Referrers/DataTable/Filter/KeywordsFromSearchEngineId.php
+++ b/plugins/Referrers/DataTable/Filter/KeywordsFromSearchEngineId.php
@@ -11,6 +11,7 @@ namespace Piwik\Plugins\Referrers\DataTable\Filter;
 use Piwik\DataTable\BaseFilter;
 use Piwik\DataTable\Row;
 use Piwik\DataTable;
+use Piwik\Plugins\Referrers\SearchEngine;
 
 class KeywordsFromSearchEngineId extends BaseFilter
 {
@@ -47,7 +48,7 @@ class KeywordsFromSearchEngineId extends BaseFilter
 
         if (!empty($subTableRow)) {
             $searchEngineUrl = $subTableRow->getMetadata('url');
-            $table->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', 'Piwik\Plugins\Referrers\getSearchEngineUrlFromKeywordAndUrl', array($searchEngineUrl)));
+            $table->queueFilter('ColumnCallbackAddMetadata', array('label', 'url',  function ($keyword, $url) { return SearchEngine::getInstance()->getBackLinkFromUrlAndKeyword($url, $keyword); }, array($searchEngineUrl)));
             $table->queueFilter(function (DataTable $table) {
                 $row = $table->getRowFromId(DataTable::ID_SUMMARY_ROW);
                 if ($row) {
diff --git a/plugins/Referrers/DataTable/Filter/SearchEnginesFromKeywordId.php b/plugins/Referrers/DataTable/Filter/SearchEnginesFromKeywordId.php
index feab8636af..6d395abba0 100644
--- a/plugins/Referrers/DataTable/Filter/SearchEnginesFromKeywordId.php
+++ b/plugins/Referrers/DataTable/Filter/SearchEnginesFromKeywordId.php
@@ -11,6 +11,7 @@ namespace Piwik\Plugins\Referrers\DataTable\Filter;
 use Piwik\DataTable\BaseFilter;
 use Piwik\DataTable\Row;
 use Piwik\DataTable;
+use Piwik\Plugins\Referrers\SearchEngine;
 
 class SearchEnginesFromKeywordId extends BaseFilter
 {
@@ -44,14 +45,14 @@ class SearchEnginesFromKeywordId extends BaseFilter
     {
         $idSubtable  = $this->idSubtable ? : $table->getId();
 
-        $table->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', 'Piwik\Plugins\Referrers\getSearchEngineUrlFromName'));
-        $table->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', 'Piwik\Plugins\Referrers\getSearchEngineLogoFromUrl'));
+        $table->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', function ($url) { return SearchEngine::getInstance()->getUrlFromName($url); }));
+        $table->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', function ($url) { return SearchEngine::getInstance()->getLogoFromUrl($url); }));
 
         // get the keyword and create the URL to the search result page
         $rootRow = $this->firstLevelKeywordTable->getRowFromIdSubDataTable($idSubtable);
         if ($rootRow) {
             $keyword = $rootRow->getColumn('label');
-            $table->queueFilter('MetadataCallbackReplace', array('url', 'Piwik\Plugins\Referrers\getSearchEngineUrlFromUrlAndKeyword', array($keyword)));
+            $table->queueFilter('MetadataCallbackReplace', array('url', function ($url, $keyword) { return SearchEngine::getInstance()->getBackLinkFromUrlAndKeyword($url, $keyword); }, array($keyword)));
             $table->queueFilter(function (DataTable $table) {
                 $row = $table->getRowFromId(DataTable::ID_SUMMARY_ROW);
                 if ($row) {
diff --git a/plugins/Referrers/Referrers.php b/plugins/Referrers/Referrers.php
index ce8667929b..d262c79224 100644
--- a/plugins/Referrers/Referrers.php
+++ b/plugins/Referrers/Referrers.php
@@ -12,6 +12,7 @@ use Piwik\ArchiveProcessor;
 use Piwik\Common;
 use Piwik\Piwik;
 use Piwik\Plugins\CoreVisualizations\Visualizations\HtmlTable;
+use Piwik\Plugins\SitesManager\SiteUrls;
 
 /**
  * @see plugins/Referrers/functions.php
@@ -31,9 +32,18 @@ class Referrers extends \Piwik\Plugin
             'Insights.addReportToOverview'      => 'addReportToInsightsOverview',
             'Live.getAllVisitorDetails'         => 'extendVisitorDetails',
             'Request.getRenamedModuleAndAction' => 'renameDeprecatedModuleAndAction',
+            'Tracker.setTrackerCacheGeneral'    => 'setTrackerCacheGeneral'
         );
     }
 
+    public function setTrackerCacheGeneral(&$cacheContent)
+    {
+        $siteUrls = new SiteUrls();
+        $urls = $siteUrls->getAllCachedSiteUrls();
+
+        return $cacheContent['allUrlsByHostAndIdSite'] = $siteUrls->groupUrlsByHost($urls);
+    }
+
     public function renameDeprecatedModuleAndAction(&$module, &$action)
     {
         if($module == 'Referers') {
diff --git a/plugins/Referrers/SearchEngine.php b/plugins/Referrers/SearchEngine.php
new file mode 100644
index 0000000000..14009a9029
--- /dev/null
+++ b/plugins/Referrers/SearchEngine.php
@@ -0,0 +1,485 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Referrers;
+
+use Piwik\Cache;
+use Piwik\Common;
+use Piwik\Option;
+use Piwik\Piwik;
+use Piwik\Singleton;
+use Piwik\UrlHelper;
+
+/**
+ * Contains methods to access search engine definition data.
+ */
+class SearchEngine extends Singleton
+{
+    const OPTION_STORAGE_NAME = 'SearchEngineDefinitions';
+
+    /** @var string location of definition file (relative to PIWIK_INCLUDE_PATH) */
+    const DEFINITION_FILE = '/vendor/piwik/searchengine-and-social-list/SearchEngines.yml';
+
+    protected $definitionList = null;
+
+    /**
+     * Returns list of search engines by URL
+     *
+     * @return array  Array of ( URL => array( searchEngineName, keywordParameter, path, charset ) )
+     */
+    public function getDefinitions()
+    {
+        $cache   = Cache::getEagerCache();
+        $cacheId = 'SearchEngine-' . self::OPTION_STORAGE_NAME;
+
+        if ($cache->contains($cacheId)) {
+            $list = $cache->fetch($cacheId);
+        } else {
+            $list = $this->loadDefinitions();
+            $cache->save($cacheId, $list);
+        }
+
+        return $list;
+    }
+
+    private function loadDefinitions()
+    {
+        if (empty($this->definitionList)) {
+            // Read first from the auto-updated list in database
+            $list = Option::get(self::OPTION_STORAGE_NAME);
+
+            if ($list) {
+                $this->definitionList = unserialize(base64_decode($list));
+            } else {
+                // Fallback to reading the bundled list
+                $yml                  = file_get_contents(PIWIK_INCLUDE_PATH . self::DEFINITION_FILE);
+                $this->definitionList = $this->loadYmlData($yml);
+                Option::set(self::OPTION_STORAGE_NAME, base64_encode(serialize($this->definitionList)));
+            }
+        }
+
+        Piwik::postEvent('Referrer.addSearchEngineUrls', array(&$this->definitionList));
+
+        $this->convertLegacyDefinitions();
+
+        return $this->definitionList;
+    }
+
+    /**
+     * @deprecated remove in 3.0
+     */
+    protected function convertLegacyDefinitions()
+    {
+        foreach ($this->definitionList as $url => $definition) {
+            if (!array_key_exists('name', $definition) && isset($definition[0]) && isset($definition[1])) {
+                $this->definitionList[$url] = array(
+                    'name' => $definition[0],
+                    'params' => $definition[1],
+                    'backlink' => @$definition[2],
+                    'charsets' => @$definition[3]
+                );
+            }
+        }
+
+    }
+
+    /**
+     * Parses the given YML string and caches the resulting definitions
+     *
+     * @param string $yml
+     * @return array
+     */
+    public function loadYmlData($yml)
+    {
+        $searchEngines = \Spyc::YAMLLoadString($yml);
+
+        $this->definitionList = $this->transformData($searchEngines);
+
+        return $this->definitionList;
+    }
+
+    protected function transformData($searchEngines)
+    {
+        $urlToInfo = array();
+        foreach ($searchEngines as $name => $info) {
+            foreach ($info as $urlDefinitions) {
+                foreach ($urlDefinitions['urls'] as $url) {
+                    $searchEngineData = $urlDefinitions;
+                    unset($searchEngineData['urls']);
+                    $searchEngineData['name'] = $name;
+                    $urlToInfo[$url]          = $searchEngineData;
+                }
+            }
+        }
+        return $urlToInfo;
+    }
+
+    /**
+     * Returns list of search engines by name
+     *
+     * @return array  Array of ( searchEngineName => URL )
+     */
+    public function getNames()
+    {
+        $cacheId   = 'SearchEngine.getSearchEngineNames';
+        $cache     = Cache::getTransientCache();
+        $nameToUrl = $cache->fetch($cacheId);
+
+        if (empty($nameToUrl)) {
+            $searchEngines = $this->getDefinitions();
+
+            $nameToUrl = array();
+            foreach ($searchEngines as $url => $info) {
+                if (!isset($nameToUrl[$info['name']])) {
+                    $nameToUrl[$info['name']] = $url;
+                }
+            }
+            $cache->save($cacheId, $nameToUrl);
+        }
+
+        return $nameToUrl;
+    }
+
+    /**
+     * Returns definitions for the given search engine host
+     *
+     * @param string $host
+     * @return array
+     */
+    public function getDefinitionByHost($host)
+    {
+        $searchEngines = $this->getDefinitions();
+
+        if (!array_key_exists($host, $searchEngines)) {
+            return array();
+        }
+
+        return $searchEngines[$host];
+    }
+
+    /**
+     * Extracts a keyword from a raw not encoded URL.
+     * Will only extract keyword if a known search engine has been detected.
+     * Returns the keyword:
+     * - in UTF8: automatically converted from other charsets when applicable
+     * - strtolowered: "QUErY test!" will return "query test!"
+     * - trimmed: extra spaces before and after are removed
+     *
+     * The function returns false when a keyword couldn't be found.
+     *     eg. if the url is "http://www.google.com/partners.html" this will return false,
+     *       as the google keyword parameter couldn't be found.
+     *
+     * @see unit tests in /tests/core/Common.test.php
+     * @param string $referrerUrl URL referrer URL, eg. $_SERVER['HTTP_REFERER']
+     * @return array|bool   false if a keyword couldn't be extracted,
+     *                        or array(
+     *                            'name' => 'Google',
+     *                            'keywords' => 'my searched keywords')
+     */
+    public function extractInformationFromUrl($referrerUrl)
+    {
+        $referrerParsed = @parse_url($referrerUrl);
+        $referrerHost   = '';
+        if (isset($referrerParsed['host'])) {
+            $referrerHost = $referrerParsed['host'];
+        }
+        if (empty($referrerHost)) {
+            return false;
+        }
+        // some search engines (eg. Bing Images) use the same domain
+        // as an existing search engine (eg. Bing), we must also use the url path
+        $referrerPath = '';
+        if (isset($referrerParsed['path'])) {
+            $referrerPath = $referrerParsed['path'];
+        }
+
+        $query = '';
+        if (isset($referrerParsed['query'])) {
+            $query = $referrerParsed['query'];
+        }
+
+        // Google Referrers URLs sometimes have the fragment which contains the keyword
+        if (!empty($referrerParsed['fragment'])) {
+            $query .= '&' . $referrerParsed['fragment'];
+        }
+
+        $referrerHost = $this->getEngineHostFromUrl($referrerHost, $referrerPath, $query);
+
+        if (empty($referrerHost)) {
+            return false;
+        }
+
+        $definitions = $this->getDefinitionByHost($referrerHost);
+
+        $searchEngineName = $definitions['name'];
+        $variableNames    = $definitions['params'];
+
+        $key = null;
+        if ($searchEngineName === 'Google Images'
+            || ($searchEngineName === 'Google' && strpos($referrerUrl, '/imgres') !== false)
+        ) {
+            if (strpos($query, '&prev') !== false) {
+                $query = urldecode(trim(UrlHelper::getParameterFromQueryString($query, 'prev')));
+                $query = str_replace('&', '&amp;', strstr($query, '?'));
+            }
+            $searchEngineName = 'Google Images';
+        } elseif ($searchEngineName === 'Google'
+            && (strpos($query, '&as_') !== false || strpos($query, 'as_') === 0)
+        ) {
+            $keys = array();
+            $key  = UrlHelper::getParameterFromQueryString($query, 'as_q');
+            if (!empty($key)) {
+                array_push($keys, $key);
+            }
+            $key = UrlHelper::getParameterFromQueryString($query, 'as_oq');
+            if (!empty($key)) {
+                array_push($keys, str_replace('+', ' OR ', $key));
+            }
+            $key = UrlHelper::getParameterFromQueryString($query, 'as_epq');
+            if (!empty($key)) {
+                array_push($keys, "\"$key\"");
+            }
+            $key = UrlHelper::getParameterFromQueryString($query, 'as_eq');
+            if (!empty($key)) {
+                array_push($keys, "-$key");
+            }
+            $key = trim(urldecode(implode(' ', $keys)));
+        }
+
+        if ($searchEngineName === 'Google') {
+            // top bar menu
+            $tbm = UrlHelper::getParameterFromQueryString($query, 'tbm');
+            switch ($tbm) {
+                case 'isch':
+                    $searchEngineName = 'Google Images';
+                    break;
+                case 'vid':
+                    $searchEngineName = 'Google Video';
+                    break;
+                case 'shop':
+                    $searchEngineName = 'Google Shopping';
+                    break;
+            }
+        }
+
+        if (empty($key)) {
+            foreach ($variableNames as $variableName) {
+                if ($variableName[0] == '/') {
+                    // regular expression match
+                    if (preg_match($variableName, $referrerUrl, $matches)) {
+                        $key = trim(urldecode($matches[1]));
+                        break;
+                    }
+                } else {
+                    // search for keywords now &vname=keyword
+                    $key = UrlHelper::getParameterFromQueryString($query, $variableName);
+                    $key = trim(urldecode($key));
+
+                    // Special cases: empty or no keywords
+                    if (empty($key)
+                        && (
+                            // Google search with no keyword
+                            ($searchEngineName == 'Google'
+                                && (empty($query) && (empty($referrerPath) || $referrerPath == '/') && empty($referrerParsed['fragment']))
+                            )
+
+                            // Yahoo search with no keyword
+                            || ($searchEngineName == 'Yahoo!'
+                                && ($referrerParsed['host'] == 'r.search.yahoo.com')
+                            )
+
+                            // empty keyword parameter
+                            || strpos($query, sprintf('&%s=', $variableName)) !== false
+                            || strpos($query, sprintf('?%s=', $variableName)) !== false
+
+                            // search engines with no keyword
+                            || $searchEngineName == 'Ixquick'
+                            || $searchEngineName == 'Google Images'
+                            || $searchEngineName == 'DuckDuckGo')
+                    ) {
+                        $key = false;
+                    }
+                    if (!empty($key)
+                        || $key === false
+                    ) {
+                        break;
+                    }
+                }
+            }
+        }
+
+        // $key === false is the special case "No keyword provided" which is a Search engine match
+        if ($key === null || $key === '') {
+            return false;
+        }
+
+        if (!empty($key)) {
+            if (!empty($definitions['charsets'])) {
+                $key = $this->convertCharset($key, $definitions['charsets']);
+            }
+            $key = Common::mb_strtolower($key);
+        }
+
+        return array(
+            'name'     => $searchEngineName,
+            'keywords' => $key,
+        );
+    }
+
+    protected function getEngineHostFromUrl($host, $path, $query)
+    {
+        $searchEngines = $this->getDefinitions();
+
+        $hostPattern = UrlHelper::getLossyUrl($host);
+        /*
+         * Try to get the best matching 'host' in definitions
+         * 1. check if host + path matches an definition
+         * 2. check if host only matches
+         * 3. check if host pattern + path matches
+         * 4. check if host pattern matches
+         * 5. special handling
+         */
+        if (array_key_exists($host . $path, $searchEngines)) {
+            $host = $host . $path;
+        } elseif (array_key_exists($host, $searchEngines)) {
+            // no need to change host
+        } elseif (array_key_exists($hostPattern . $path, $searchEngines)) {
+            $host = $hostPattern . $path;
+        } elseif (array_key_exists($hostPattern, $searchEngines)) {
+            $host = $hostPattern;
+        } elseif (!array_key_exists($host, $searchEngines)) {
+            if (!strncmp($query, 'cx=partner-pub-', 15)) {
+                // Google custom search engine
+                $host = 'google.com/cse';
+            } elseif (!strncmp($path, '/pemonitorhosted/ws/results/', 28)) {
+                // private-label search powered by InfoSpace Metasearch
+                $host = 'wsdsold.infospace.com';
+            } elseif (strpos($host, '.images.search.yahoo.com') != false) {
+                // Yahoo! Images
+                $host = 'images.search.yahoo.com';
+            } elseif (strpos($host, '.search.yahoo.com') != false) {
+                // Yahoo!
+                $host = 'search.yahoo.com';
+            } else {
+                return false;
+            }
+        }
+
+        return $host;
+    }
+
+    /**
+     * Tries to convert the given string from one of the given charsets to UTF-8
+     * @param string $string
+     * @param array $charsets
+     * @return string
+     */
+    protected function convertCharset($string, $charsets)
+    {
+        if (function_exists('iconv')
+            && !empty($charsets)
+        ) {
+            $charset = $charsets[0];
+            if (count($charsets) > 1
+                && function_exists('mb_detect_encoding')
+            ) {
+                $charset = mb_detect_encoding($string, $charsets);
+                if ($charset === false) {
+                    $charset = $charsets[0];
+                }
+            }
+
+            $newKey = @iconv($charset, 'UTF-8//IGNORE', $string);
+            if (!empty($newKey)) {
+                $string = $newKey;
+            }
+        }
+
+        return $string;
+    }
+
+    /**
+     * Return search engine URL by name
+     *
+     * @see core/DataFiles/SearchEnginges.php
+     *
+     * @param string $name
+     * @return string URL
+     */
+    public function getUrlFromName($name)
+    {
+        $searchEngineNames = $this->getNames();
+        if (isset($searchEngineNames[$name])) {
+            $url = 'http://' . $searchEngineNames[$name];
+        } else {
+            $url = 'URL unknown!';
+        }
+        return $url;
+    }
+
+    /**
+     * Return search engine host in URL
+     *
+     * @param string $url
+     * @return string host
+     */
+    private function getHostFromUrl($url)
+    {
+        if (strpos($url, '//')) {
+            $url = substr($url, strpos($url, '//') + 2);
+        }
+        if (($p = strpos($url, '/')) !== false) {
+            $url = substr($url, 0, $p);
+        }
+        return $url;
+    }
+
+
+    /**
+     * Return search engine logo path by URL
+     *
+     * @param string $url
+     * @return string path
+     * @see plugins/Referrers/images/searchEnginges/
+     */
+    public function getLogoFromUrl($url)
+    {
+        $pathInPiwik  = 'plugins/Referrers/images/searchEngines/%s.png';
+        $pathWithCode = sprintf($pathInPiwik, $this->getHostFromUrl($url));
+        $absolutePath = PIWIK_INCLUDE_PATH . '/' . $pathWithCode;
+        if (file_exists($absolutePath)) {
+            return $pathWithCode;
+        }
+        return sprintf($pathInPiwik, 'xx');
+    }
+
+    /**
+     * Return search engine URL for URL and keyword
+     *
+     * @see core/DataFiles/SearchEnginges.php
+     *
+     * @param string $url Domain name, e.g., search.piwik.org
+     * @param string $keyword Keyword, e.g., web+analytics
+     * @return string URL, e.g., http://search.piwik.org/q=web+analytics
+     */
+    public function getBackLinkFromUrlAndKeyword($url, $keyword)
+    {
+        if ($keyword === API::LABEL_KEYWORD_NOT_DEFINED) {
+            return 'http://piwik.org/faq/general/#faq_144';
+        }
+        $keyword = urlencode($keyword);
+        $keyword = str_replace(urlencode('+'), urlencode(' '), $keyword);
+        $host    = substr($url, strpos($url, '//') + 2);
+        $definition = $this->getDefinitionByHost($host);
+        if (empty($definition['backlink'])) {
+            return false;
+        }
+        $path = str_replace("{k}", $keyword, $definition['backlink']);
+        return $url . (substr($url, -1) != '/' ? '/' : '') . $path;
+    }
+}
diff --git a/plugins/Referrers/Social.php b/plugins/Referrers/Social.php
new file mode 100644
index 0000000000..71badc3357
--- /dev/null
+++ b/plugins/Referrers/Social.php
@@ -0,0 +1,181 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Referrers;
+use Piwik\Cache;
+use Piwik\Common;
+use Piwik\Option;
+use Piwik\Piwik;
+use Piwik\Singleton;
+use Piwik\UrlHelper;
+
+/**
+ * Contains methods to access search engine definition data.
+ */
+class Social extends Singleton
+{
+    const OPTION_STORAGE_NAME = 'SocialDefinitions';
+
+    /** @var string location of definition file (relative to PIWIK_INCLUDE_PATH) */
+    const DEFINITION_FILE = '/vendor/piwik/searchengine-and-social-list/Socials.yml';
+
+    protected $definitionList = null;
+
+    /**
+     * Returns list of search engines by URL
+     *
+     * @return array  Array of ( URL => array( searchEngineName, keywordParameter, path, charset ) )
+     */
+    public function getDefinitions()
+    {
+        $cache = Cache::getEagerCache();
+        $cacheId = 'Social-' . self::OPTION_STORAGE_NAME;
+
+        if ($cache->contains($cacheId)) {
+            $list = $cache->fetch($cacheId);
+        } else {
+            $list = $this->loadDefinitions();
+            $cache->save($cacheId, $list);
+        }
+
+        return $list;
+    }
+
+    private function loadDefinitions()
+    {
+        if ($this->definitionList === null) {
+            // Read first from the auto-updated list in database
+            $list = Option::get(self::OPTION_STORAGE_NAME);
+
+            if ($list) {
+                $this->definitionList = unserialize(base64_decode($list));
+            } else {
+                // Fallback to reading the bundled list
+                $yml = file_get_contents(PIWIK_INCLUDE_PATH . self::DEFINITION_FILE);
+                $this->definitionList = $this->loadYmlData($yml);
+                Option::set(self::OPTION_STORAGE_NAME, base64_encode(serialize($this->definitionList)));
+            }
+        }
+
+        Piwik::postEvent('Referrer.addSocialUrls', array(&$this->definitionList));
+
+        return $this->definitionList;
+    }
+
+    /**
+     * Parses the given YML string and caches the resulting definitions
+     *
+     * @param string $yml
+     * @return array
+     */
+    public function loadYmlData($yml)
+    {
+        $searchEngines = \Spyc::YAMLLoadString($yml);
+
+        $this->definitionList = $this->transformData($searchEngines);
+
+        return $this->definitionList;
+    }
+
+    protected function transformData($socials)
+    {
+        $urlToName = array();
+        foreach ($socials as $name => $urls) {
+            foreach ($urls as $url) {
+                $urlToName[$url] = $name;
+            }
+        }
+        return $urlToName;
+    }
+
+    /**
+     * Returns true if a URL belongs to a social network, false if otherwise.
+     *
+     * @param string $url The URL to check.
+     * @param string|bool $socialName The social network's name to check for, or false to check
+     *                                 for any.
+     * @return bool
+     */
+    public function isSocialUrl($url, $socialName = false)
+    {
+        foreach ($this->getDefinitions() as $domain => $name) {
+
+            if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url) && ($socialName === false || $name == $socialName)) {
+
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+
+    /**
+     * Get's social network name from URL.
+     *
+     * @param string $url
+     * @return string
+     */
+    public function getSocialNetworkFromDomain($url)
+    {
+        foreach ($this->getDefinitions() as $domain => $name) {
+
+            if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url)) {
+
+                return $name;
+            }
+        }
+
+        return Piwik::translate('General_Unknown');
+    }
+
+    /**
+     * Returns the main url of the social network the given url matches
+     *
+     * @param string  $url
+     *
+     * @return string
+     */
+    public function getMainUrl($url)
+    {
+        $social  = $this->getSocialNetworkFromDomain($url);
+        foreach ($this->getDefinitions() as $domain => $name) {
+
+            if ($name == $social) {
+
+                return $domain;
+            }
+        }
+        return $url;
+    }
+
+
+    /**
+     * Return social network logo path by URL
+     *
+     * @param string $domain
+     * @return string path
+     * @see plugins/Referrers/images/socials/
+     */
+    public function getLogoFromUrl($domain)
+    {
+        $social = $this->getSocialNetworkFromDomain($domain);
+        $socialNetworks = $this->getDefinitions();
+
+        $filePattern = 'plugins/Referrers/images/socials/%s.png';
+
+        $socialDomains = array_keys($socialNetworks, $social);
+        foreach ($socialDomains as $domain) {
+            if (file_exists(PIWIK_INCLUDE_PATH . '/' . sprintf($filePattern, $domain))) {
+                return sprintf($filePattern, $domain);
+            }
+        }
+
+        return sprintf($filePattern, 'xx');
+    }
+}
diff --git a/plugins/Referrers/Tasks.php b/plugins/Referrers/Tasks.php
new file mode 100644
index 0000000000..da9cad1321
--- /dev/null
+++ b/plugins/Referrers/Tasks.php
@@ -0,0 +1,54 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ *
+ */
+namespace Piwik\Plugins\Referrers;
+
+
+use Piwik\Http;
+use Piwik\Option;
+
+class Tasks extends \Piwik\Plugin\Tasks
+{
+    public function schedule()
+    {
+        $this->weekly('updateSearchEngines');
+        $this->weekly('updateSocials');
+    }
+
+    /**
+     * Update the search engine definitions
+     *
+     * @see https://github.com/piwik/searchengine-and-social-list
+     */
+    public function updateSearchEngines()
+    {
+        $url = 'https://raw.githubusercontent.com/piwik/searchengine-and-social-list/master/SearchEngines.yml';
+        $list = Http::sendHttpRequest($url, 30);
+        $searchEngines = SearchEngine::getInstance()->loadYmlData($list);
+        if (count($searchEngines) < 200) {
+            return;
+        }
+        Option::set(SearchEngine::OPTION_STORAGE_NAME, base64_encode(serialize($searchEngines)));
+    }
+
+    /**
+     * Update the social definitions
+     *
+     * @see https://github.com/piwik/searchengine-and-social-list
+     */
+    public function updateSocials()
+    {
+        $url = 'https://raw.githubusercontent.com/piwik/searchengine-and-social-list/master/Socials.yml';
+        $list = Http::sendHttpRequest($url, 30);
+        $socials = Social::getInstance()->loadYmlData($list);
+        if (count($socials) < 50) {
+            return;
+        }
+        Option::set(Social::OPTION_STORAGE_NAME, base64_encode(serialize($socials)));
+    }
+}
\ No newline at end of file
diff --git a/plugins/Referrers/Visitor.php b/plugins/Referrers/Visitor.php
index 308c7b08f8..b10b9b953d 100644
--- a/plugins/Referrers/Visitor.php
+++ b/plugins/Referrers/Visitor.php
@@ -62,7 +62,7 @@ class Visitor
             ) {
                 $refUrl = @parse_url($this->details['referer_url']);
                 if (isset($refUrl['host'])) {
-                    $url = getSearchEngineUrlFromUrlAndKeyword('http://google.com', $this->getKeyword());
+                    $url = SearchEngine::getInstance()->getBackLinkFromUrlAndKeyword('http://google.com', $this->getKeyword());
                     $url = str_replace('google.com', $refUrl['host'], $url);
 
                     return $url;
@@ -109,7 +109,7 @@ class Visitor
             && !empty($this->details['referer_name'])
         ) {
 
-            return getSearchEngineUrlFromName($this->details['referer_name']);
+            return SearchEngine::getInstance()->getUrlFromName($this->details['referer_name']);
         }
 
         return null;
@@ -121,7 +121,7 @@ class Visitor
 
         if (!is_null($searchEngineUrl)) {
 
-            return getSearchEngineLogoFromUrl($searchEngineUrl);
+            return SearchEngine::getInstance()->getLogoFromUrl($searchEngineUrl);
         }
 
         return null;
diff --git a/plugins/Referrers/functions.php b/plugins/Referrers/functions.php
index e0fee30833..c91ebc816d 100644
--- a/plugins/Referrers/functions.php
+++ b/plugins/Referrers/functions.php
@@ -28,194 +28,6 @@ function getPathFromUrl($url)
 }
 
 /**
- * Returns the main url of the social network the given url matches
- *
- * @param string  $url
- *
- * @return string
- */
-function getSocialMainUrl($url)
-{
-    $social  = getSocialNetworkFromDomain($url);
-    foreach (Common::getSocialUrls() as $domain => $name) {
-
-        if ($name == $social) {
-
-            return $domain;
-        }
-    }
-    return $url;
-}
-
-/**
- * Get's social network name from URL.
- *
- * @param string $url
- * @return string
- */
-function getSocialNetworkFromDomain($url)
-{
-    foreach (Common::getSocialUrls() as $domain => $name) {
-
-        if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url)) {
-
-            return $name;
-        }
-    }
-
-    return Piwik::translate('General_Unknown');
-}
-
-/**
- * Returns true if a URL belongs to a social network, false if otherwise.
- *
- * @param string $url The URL to check.
- * @param string|bool $socialName The social network's name to check for, or false to check
- *                                 for any.
- * @return bool
- */
-function isSocialUrl($url, $socialName = false)
-{
-    foreach (Common::getSocialUrls() as $domain => $name) {
-
-        if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url) && ($socialName === false || $name == $socialName)) {
-
-            return true;
-        }
-    }
-
-    return false;
-}
-
-/**
- * Return social network logo path by URL
- *
- * @param string $domain
- * @return string path
- * @see plugins/Referrers/images/socials/
- */
-function getSocialsLogoFromUrl($domain)
-{
-    $social = getSocialNetworkFromDomain($domain);
-    $socialNetworks = Common::getSocialUrls();
-
-    $filePattern = 'plugins/Referrers/images/socials/%s.png';
-
-    foreach ($socialNetworks as $domainKey => $name) {
-        if ($social == $socialNetworks[$domainKey] && file_exists(PIWIK_INCLUDE_PATH . '/' . sprintf($filePattern, $domainKey))) {
-            return sprintf($filePattern, $domainKey);
-        }
-    }
-
-    return sprintf($filePattern, 'xx');
-}
-
-/**
- * Return search engine URL by name
- *
- * @see core/DataFiles/SearchEnginges.php
- *
- * @param string $name
- * @return string URL
- */
-function getSearchEngineUrlFromName($name)
-{
-    $searchEngineNames = Common::getSearchEngineNames();
-    if (isset($searchEngineNames[$name])) {
-        $url = 'http://' . $searchEngineNames[$name];
-    } else {
-        $url = 'URL unknown!';
-    }
-    return $url;
-}
-
-/**
- * Return search engine host in URL
- *
- * @param string $url
- * @return string host
- */
-function getSearchEngineHostFromUrl($url)
-{
-    if (strpos($url, '//')) {
-        $url = substr($url, strpos($url, '//') + 2);
-    }
-    if (($p = strpos($url, '/')) !== false) {
-        $url = substr($url, 0, $p);
-    }
-    return $url;
-}
-
-/**
- * Return search engine logo path by URL
- *
- * @param string $url
- * @return string path
- * @see plugins/Referrers/images/searchEnginges/
- */
-function getSearchEngineLogoFromUrl($url)
-{
-    $pathInPiwik = 'plugins/Referrers/images/searchEngines/%s.png';
-    $pathWithCode = sprintf($pathInPiwik, getSearchEngineHostFromUrl($url));
-    $absolutePath = PIWIK_INCLUDE_PATH . '/' . $pathWithCode;
-    if (file_exists($absolutePath)) {
-        return $pathWithCode;
-    }
-    return sprintf($pathInPiwik, 'xx');
-}
-
-/**
- * Return search engine host and path in URL
- *
- * @param string $url
- * @return string host
- */
-function getSearchEngineHostPathFromUrl($url)
-{
-    $url = substr($url, strpos($url, '//') + 2);
-    return $url;
-}
-
-/**
- * Return search engine URL for URL and keyword
- *
- * @see core/DataFiles/SearchEnginges.php
- *
- * @param string $url Domain name, e.g., search.piwik.org
- * @param string $keyword Keyword, e.g., web+analytics
- * @return string URL, e.g., http://search.piwik.org/q=web+analytics
- */
-function getSearchEngineUrlFromUrlAndKeyword($url, $keyword)
-{
-    if ($keyword === API::LABEL_KEYWORD_NOT_DEFINED) {
-        return 'http://piwik.org/faq/general/#faq_144';
-    }
-    $searchEngineUrls = Common::getSearchEngineUrls();
-    $keyword = urlencode($keyword);
-    $keyword = str_replace(urlencode('+'), urlencode(' '), $keyword);
-    $path = @$searchEngineUrls[getSearchEngineHostPathFromUrl($url)][2];
-    if (empty($path)) {
-        return false;
-    }
-    $path = str_replace("{k}", $keyword, $path);
-    return $url . (substr($url, -1) != '/' ? '/' : '') . $path;
-}
-
-/**
- * Return search engine URL for keyword and URL
- *
- * @see \Piwik\Plugins\Referrers\getSearchEngineUrlFromUrlAndKeyword
- *
- * @param string $keyword Keyword, e.g., web+analytics
- * @param string $url Domain name, e.g., search.piwik.org
- * @return string URL, e.g., http://search.piwik.org/q=web+analytics
- */
-function getSearchEngineUrlFromKeywordAndUrl($keyword, $url)
-{
-    return getSearchEngineUrlFromUrlAndKeyword($url, $keyword);
-}
-
-/**
  * Return translated referrer type
  *
  * @param string $label
@@ -223,7 +35,6 @@ function getSearchEngineUrlFromKeywordAndUrl($keyword, $url)
  */
 function getReferrerTypeLabel($label)
 {
-    $indexTranslation = '';
     switch ($label) {
         case Common::REFERRER_TYPE_DIRECT_ENTRY:
             $indexTranslation = 'Referrers_DirectEntry';
diff --git a/plugins/Referrers/lang/cs.json b/plugins/Referrers/lang/cs.json
index 740d25a38b..052a054161 100644
--- a/plugins/Referrers/lang/cs.json
+++ b/plugins/Referrers/lang/cs.json
@@ -3,9 +3,9 @@
         "AllReferrersReportDocumentation": "Toto hlášení zobrazuje všechny referrery v jednom uceleném hlášení, kde zobrazuje všechny webové stránky, klíčová slova a kampaně, které vaši návštěvníci při hledání vašich webových stránek použili.",
         "Campaigns": "Kampaně",
         "CampaignsDocumentation": "Návštěvníci, kteří přišli na vaše webové stránky jako výsledek kampaně. %s Pro více informací si prohlédněte hlášení %s.",
-        "CampaignsReportDocumentation": "Toto hlášení zobrazuje, jaké kampaně přivedly návštěvníky na vaše stránky. %s Pro více informací o sledování kampaní si %spřečtěte dokumentaci kampaní na piwik.org%s.",
+        "CampaignsReportDocumentation": "Toto hlášení zobrazuje, které kampaně přivedly návštěvníky na vaše stránky. %s Pro více informací o sledování kampaní si %spřečtěte dokumentaci kampaní na piwik.org%s.",
         "ColumnCampaign": "Kampaň",
-        "ColumnSearchEngine": "Vyhledávací stroj",
+        "ColumnSearchEngine": "Vyhledávač",
         "ColumnSocial": "Sociální síť",
         "ColumnWebsite": "Web stránky",
         "ColumnWebsitePage": "Web stránka",
@@ -15,7 +15,7 @@
         "DistinctCampaigns": "jedinečné kampaně",
         "DistinctKeywords": "jedinečná klíčová slova",
         "DistinctSearchEngines": "jedinečné vyhledávače",
-        "DistinctWebsites": "jedinečná weby",
+        "DistinctWebsites": "jedinečné weby",
         "EvolutionDocumentation": "Toto je přehled referrerů, které vedly návštěvníky na vaše webové stránky.",
         "EvolutionDocumentationMoreInfo": "Pro více informací o rozdílných typech referrerů nahlédněte do dokumentace tabulky %s.",
         "Keywords": "Klíčová slova",
@@ -26,13 +26,13 @@
         "Referrers": "Doporučení",
         "ReferrersOverview": "Přehled referrerů",
         "ReferrerTypes": "Typy referrerů",
-        "SearchEngines": "Vyhledávacích strojů",
+        "SearchEngines": "Vyhledávače",
         "SearchEnginesDocumentation": "Návštěvník byl na vaše stránky odkázán vyhledávačem. %s Pro více informací si prohlédněte hlášení %s.",
         "SearchEnginesReportDocumentation": "Toto hlášení zobrazuje, které vyhledávače odkázali návštěvníky na vaše stránky. %s Kliknutím na řádek v tabulce zobrazíte, co uživatelé na daném vyhledávači hledali.",
-        "SocialFooterMessage": "Toto je podmnožina hlášení webových stránek zobrazeného vlevo. Filtruje ostatní stránky, takže můžete přímo porovnat referrery ze sociálních sítí.",
+        "SocialFooterMessage": "Toto je podmnožina hlášení zobrazeného vlevo. Filtruje ostatní stránky, takže můžete přímo porovnat referrery ze sociálních sítí.",
         "Socials": "Sociální sítě",
         "SocialsReportDocumentation": "Toto hlášení zobrazuje, jaké sociální sítě přivedly návštěvníky na vaše stránky. <br\/> Kliknutím na Kliknutím na řádek tabulky zobrazíte stránky sociální sítě, ze kterých návštěvníci přišli.",
-        "SubmenuSearchEngines": "Vyhledávače & klíčová slova",
+        "SubmenuSearchEngines": "Vyhledávače a klíčová slova",
         "SubmenuWebsites": "Web",
         "Type": "Typ refereru",
         "TypeCampaigns": "%s z kampaní",
diff --git a/plugins/Referrers/lang/ko.json b/plugins/Referrers/lang/ko.json
index f5f309c895..28c132386a 100644
--- a/plugins/Referrers/lang/ko.json
+++ b/plugins/Referrers/lang/ko.json
@@ -20,10 +20,12 @@
         "EvolutionDocumentationMoreInfo": "다른 참조 유형에 대한 자세한 내용은 %s 테이블의 문서를 참조하세요.",
         "Keywords": "검색어",
         "KeywordsReportDocumentation": "이 보고서는 사용자가 웹사이트를 방문하는 데 사용하는 검색어를 나타냅니다. %s 테이블의 행을 클릭하면 검색어에 조회된 검색 엔진의 분포를 볼 수 있습니다.",
+        "PluginDescription": "리퍼터 데이터 보고서: 검색 엔진, 키워드, 웹사이트, 캠페인, 소셜 미디어, 직접 입력",
         "Referrer": "리퍼러",
         "ReferrerName": "리퍼러 이름",
         "Referrers": "참조",
         "ReferrersOverview": "리퍼러 개요",
+        "ReferrerTypes": "리퍼러 타입",
         "SearchEngines": "검색엔진",
         "SearchEnginesDocumentation": "검색 엔진을 참조하여 웹사이트로 접속한 방문자입니다. 상세 보고서는 %s 여기 %s에 있습니다.",
         "SearchEnginesReportDocumentation": "이 보고서는 어떤 검색 엔진으로 웹사이트에 사용자가 유입되었는 지를 보여줍니다. %s 테이블의 행을 클릭하면 사용자가 특정 검색 엔진을 사용하여 무엇을 찾고 있었는지를 확인할 수 있습니다.",
@@ -47,6 +49,7 @@
         "WidgetExternalWebsites": "외부 웹사이트 목록",
         "WidgetGetAll": "모든 리퍼러",
         "WidgetSocials": "소셜 네트워크의 목록",
-        "WidgetTopKeywordsForPages": "패지 URL의 상위 키워드"
+        "WidgetTopKeywordsForPages": "패지 URL의 상위 키워드",
+        "XPercentOfVisits": "%s 번"
     }
 }
\ No newline at end of file
diff --git a/plugins/Referrers/tests/Integration/Columns/ReferrerNameTest.php b/plugins/Referrers/tests/Integration/Columns/ReferrerNameTest.php
new file mode 100644
index 0000000000..db34a25c46
--- /dev/null
+++ b/plugins/Referrers/tests/Integration/Columns/ReferrerNameTest.php
@@ -0,0 +1,130 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\Referrers\tests\Integration\Columns;
+
+use Piwik\Common;
+use Piwik\Plugins\Referrers\Columns\ReferrerName;
+use Piwik\Plugins\Referrers\Columns\ReferrerType;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+use Piwik\Tracker\Cache;
+use Piwik\Tracker\Request;
+use Piwik\Tracker\Visit\VisitProperties;
+use Piwik\Tracker\Visitor;
+
+/**
+ * @group Referrers
+ * @group ReferrerNameTest
+ * @group ReferrerName
+ * @group Plugins
+ */
+class ReferrerNameTest extends IntegrationTestCase
+{
+    /**
+     * @var ReferrerType
+     */
+    private $referrerName;
+    private $idSite1 = 1;
+    private $idSite2 = 2;
+    private $idSite3 = 3;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        Cache::clearCacheGeneral();
+
+        $date = '2012-01-01 00:00:00';
+        $ecommerce = false;
+
+        Fixture::createWebsite($date, $ecommerce, $name = 'test1', $url = 'http://piwik.org/foo/bar');
+        Fixture::createWebsite($date, $ecommerce, $name = 'test2', $url = 'http://piwik.org/');
+        Fixture::createWebsite($date, $ecommerce, $name = 'test3', $url = 'http://piwik.pro/');
+
+        $this->referrerName = new ReferrerName();
+    }
+
+    public function tearDown()
+    {
+        // clean up your test here if needed
+        Cache::clearCacheGeneral();
+
+        parent::tearDown();
+    }
+
+    /**
+     * @dataProvider getReferrerUrls
+     */
+    public function test_onNewVisit_shouldDetectCorrectReferrerType($expectedType, $idSite, $url, $referrerUrl)
+    {
+        $request = $this->getRequest(array('idsite' => $idSite, 'url' => $url, 'urlref' => $referrerUrl));
+        $type = $this->referrerName->onNewVisit($request, $this->getNewVisitor(), $action = null);
+
+        $this->assertSame($expectedType, $type);
+    }
+
+    public function getReferrerUrls()
+    {
+        $url = 'http://piwik.org/foo/bar';
+        $referrer = 'http://piwik.org';
+
+        $directEntryReferrerName = '';
+
+        return array(
+            // domain matches but path does not match for idsite1
+            array('piwik.org',              $this->idSite1, $url, $referrer),
+            array('piwik.org',              $this->idSite1, $url, $referrer . '/'),
+            // idSite2 matches any piwik.org path so this is a direct entry for it
+            array($directEntryReferrerName, $this->idSite2, $url, $referrer),
+            array($directEntryReferrerName, $this->idSite2, $url, $referrer . '/'),
+            // idSite3 has different domain so it is coming from different website
+            array('piwik.org',              $this->idSite3, $url, $referrer),
+            array('piwik.org',              $this->idSite3, $url, $referrer . '/'),
+
+            array($directEntryReferrerName, $this->idSite1, $url, $referrer . '/foo/bar/baz'),
+            array($directEntryReferrerName, $this->idSite1, $url, $referrer . '/foo/bar/baz/'),
+            array($directEntryReferrerName, $this->idSite1, $url, $referrer . '/foo/bar/baz?x=5'),
+            array($directEntryReferrerName, $this->idSite1, $url, $referrer . '/fOo/BaR/baz?x=5'),
+            // /not/xyz belongs to different website
+            array('piwik.org',              $this->idSite1, $url, $referrer . '/not/xyz'),
+            array($directEntryReferrerName, $this->idSite2, $url, $referrer . '/not/xyz'),
+
+            // /foo/bar/baz belongs to different website
+            array('piwik.org/foo/bar',      $this->idSite2, $url, $referrer . '/foo/bar/baz'),
+            array('piwik.org/foo/bar',      $this->idSite3, $url, $referrer . '/foo/bar'),
+            array('piwik.org/foo/bar',      $this->idSite3, $url, $referrer . '/fOo/BaR'),
+
+            // should detect campaign independent of domain / path
+            array('test',                   $this->idSite1, $url . '?pk_campaign=test', $referrer),
+            array('testfoobar',             $this->idSite2, $url . '?pk_campaign=testfoobar', $referrer),
+            array('test',                   $this->idSite3, $url . '?pk_campaign=test', $referrer),
+
+            array('Google',                 $this->idSite3, $url, 'http://google.com/search?q=piwik'),
+
+            // testing case for backwards compatibility where url has same domain as urlref but the domain is not known to any website
+            array($directEntryReferrerName, $this->idSite3, 'http://example.com/foo', 'http://example.com/bar'),
+            array($directEntryReferrerName, $this->idSite3, 'http://example.com/foo', 'http://example.com'),
+            array($directEntryReferrerName, $this->idSite3, 'http://example.com',     'http://example.com/bar'),
+
+            // testing case where domain of referrer is not known to any site but neither is the URL, url != urlref
+            array('example.com',            $this->idSite3, 'http://example.org',     'http://example.com/bar'),
+        );
+    }
+
+    private function getRequest($params)
+    {
+        return new Request($params);
+    }
+
+    private function getNewVisitor()
+    {
+        return new Visitor(new VisitProperties());
+    }
+
+}
diff --git a/plugins/Referrers/tests/Integration/Columns/ReferrerTypeTest.php b/plugins/Referrers/tests/Integration/Columns/ReferrerTypeTest.php
new file mode 100644
index 0000000000..5925dbc5c0
--- /dev/null
+++ b/plugins/Referrers/tests/Integration/Columns/ReferrerTypeTest.php
@@ -0,0 +1,128 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\Referrers\tests\Integration\Columns;
+
+use Piwik\Common;
+use Piwik\Plugins\Referrers\Columns\ReferrerType;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+use Piwik\Tracker\Cache;
+use Piwik\Tracker\Request;
+use Piwik\Tracker\Visit\VisitProperties;
+use Piwik\Tracker\Visitor;
+
+/**
+ * @group Referrers
+ * @group ReferrerTypeTest
+ * @group ReferrerType
+ * @group Plugins
+ */
+class ReferrerTypeTest extends IntegrationTestCase
+{
+    /**
+     * @var ReferrerType
+     */
+    private $referrerType;
+    private $idSite1 = 1;
+    private $idSite2 = 2;
+    private $idSite3 = 3;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        Cache::clearCacheGeneral();
+
+        $date = '2012-01-01 00:00:00';
+        $ecommerce = false;
+
+        Fixture::createWebsite($date, $ecommerce, $name = 'test1', $url = 'http://piwik.org/foo/bar');
+        Fixture::createWebsite($date, $ecommerce, $name = 'test2', $url = 'http://piwik.org/');
+        Fixture::createWebsite($date, $ecommerce, $name = 'test3', $url = 'http://piwik.pro/');
+
+        $this->referrerType = new ReferrerType();
+    }
+
+    public function tearDown()
+    {
+        // clean up your test here if needed
+        Cache::clearCacheGeneral();
+
+        parent::tearDown();
+    }
+
+    /**
+     * @dataProvider getReferrerUrls
+     */
+    public function test_onNewVisit_shouldDetectCorrectReferrerType($expectedType, $idSite, $url, $referrerUrl)
+    {
+        $request = $this->getRequest(array('idsite' => $idSite, 'url' => $url, 'urlref' => $referrerUrl));
+        $type = $this->referrerType->onNewVisit($request, $this->getNewVisitor(), $action = null);
+
+        $this->assertSame($expectedType, $type);
+    }
+
+    public function getReferrerUrls()
+    {
+        $url = 'http://piwik.org/foo/bar';
+        $referrer = 'http://piwik.org';
+
+        // $expectedType,                             $idSite,        $url, $referrerUrl
+        return array(
+            // domain matches but path does not match for idsite1
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite1, $url, $referrer),
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite1, $url, $referrer . '/'),
+            // idSite2 matches any piwik.org path so this is a direct entry for it
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite2, $url, $referrer),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite2, $url, $referrer . '/'),
+            // idSite3 has different domain so it is coming from different website
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite3, $url, $referrer),
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite3, $url, $referrer . '/'),
+
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite1, $url, $referrer . '/foo/bar/baz'),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite1, $url, $referrer . '/foo/bar/baz/'),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite1, $url, $referrer . '/foo/bar/baz?x=5'),
+            // /not/xyz belongs to different website
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite1, $url, $referrer . '/not/xyz'),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite2, $url, $referrer . '/not/xyz'),
+
+            // /foo/bar/baz belongs to different website
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite2, $url, $referrer . '/foo/bar/baz'),
+
+            // website as it is from different domain anyway
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite3, $url, $referrer . '/foo/bar/baz'),
+
+            // should detect campaign independent of domain / path
+            array(Common::REFERRER_TYPE_CAMPAIGN,     $this->idSite1, $url . '?pk_campaign=test', $referrer),
+            array(Common::REFERRER_TYPE_CAMPAIGN,     $this->idSite2, $url . '?pk_campaign=test', $referrer),
+            array(Common::REFERRER_TYPE_CAMPAIGN,     $this->idSite3, $url . '?pk_campaign=test', $referrer),
+
+            array(Common::REFERRER_TYPE_SEARCH_ENGINE, $this->idSite3, $url, 'http://google.com/search?q=piwik'),
+
+            // testing case for backwards compatibility where url has same domain as urlref but the domain is not known to any website
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite3, 'http://example.com/foo', 'http://example.com/bar'),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite3, 'http://example.com/foo', 'http://example.com'),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite3, 'http://example.com', 'http://example.com/bar'),
+
+            // testing case where domain of referrer is not known to any site but neither is the URL, url != urlref
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite3, 'http://example.org', 'http://example.com/bar'),
+        );
+    }
+
+    private function getRequest($params)
+    {
+        return new Request($params);
+    }
+
+    private function getNewVisitor()
+    {
+        return new Visitor(new VisitProperties());
+    }
+
+}
diff --git a/plugins/Referrers/tests/System/expected/test_Referrers_getReferrerType__API.getProcessedReport_day.xml b/plugins/Referrers/tests/System/expected/test_Referrers_getReferrerType__API.getProcessedReport_day.xml
index edc1bd40ff..58601dc41b 100644
--- a/plugins/Referrers/tests/System/expected/test_Referrers_getReferrerType__API.getProcessedReport_day.xml
+++ b/plugins/Referrers/tests/System/expected/test_Referrers_getReferrerType__API.getProcessedReport_day.xml
@@ -874,7 +874,7 @@
 		<result prettyDate="Tuesday, February 2, 2010"/>
 		<result prettyDate="Wednesday, February 3, 2010"/>
 		<result prettyDate="Thursday, February 4, 2010">
-		<row>
+			<row>
 				<idsubdatatable>2</idsubdatatable>
 			</row>
 			<row>
diff --git a/plugins/Referrers/tests/Unit/ReferrersTest.php b/plugins/Referrers/tests/Unit/ReferrersTest.php
index 909e6bf65e..e5dea866e9 100644
--- a/plugins/Referrers/tests/Unit/ReferrersTest.php
+++ b/plugins/Referrers/tests/Unit/ReferrersTest.php
@@ -11,217 +11,15 @@ namespace Piwik\Plugins\Referrers\tests;
 use Piwik\DataTable;
 use Piwik\DataTable\Row;
 use Piwik\Period;
+use Piwik\Plugins\Referrers\SearchEngine;
 
 require_once PIWIK_INCLUDE_PATH . '/plugins/Referrers/Referrers.php';
 
+/**
+ * @group Plugin
+ */
 class ReferrersTest extends \PHPUnit_Framework_TestCase
 {
-    /**
-     * Dataprovider serving all search engine data
-     */
-    public function getSearchEngines()
-    {
-        include PIWIK_PATH_TEST_TO_ROOT . '/core/DataFiles/SearchEngines.php';
-
-        $searchEngines = array();
-        foreach ($GLOBALS['Piwik_SearchEngines'] as $url => $searchEngine) {
-            $searchEngines[] = array($url, $searchEngine);
-        }
-        return $searchEngines;
-    }
-
-    /**
-     * search engine has at least one keyword
-     *
-     * @group Plugins
-     *
-     * @dataProvider getSearchEngines
-     */
-    public function testMissingSearchEngineKeyword($url, $searchEngine)
-    {
-        // Get list of search engines and first appearing URL
-        static $searchEngines = array();
-
-        $name = parse_url('http://' . $url);
-        if (!array_key_exists($searchEngine[0], $searchEngines)) {
-            $searchEngines[$searchEngine[0]] = $url;
-
-            $this->assertTrue(!empty($searchEngine[1]), $name['host']);
-        }
-    }
-
-    /**
-     * search engine is defined in DataFiles/SearchEngines.php but there's no favicon
-     *
-     * @group Plugins
-     *
-     * @dataProvider getSearchEngines
-     */
-    public function testMissingSearchEngineIcons($url, $searchEngine)
-    {
-        // Get list of existing favicons
-        $favicons = scandir(PIWIK_PATH_TEST_TO_ROOT . '/plugins/Referrers/images/searchEngines/');
-
-        // Get list of search engines and first appearing URL
-        static $searchEngines = array();
-
-        $name = parse_url('http://' . $url);
-        if (!array_key_exists($searchEngine[0], $searchEngines)) {
-            $searchEngines[$searchEngine[0]] = $url;
-
-            $this->assertTrue(in_array($name['host'] . '.png', $favicons), $name['host']);
-        }
-    }
-
-    /**
-     * favicon exists but there's no corresponding search engine defined in DataFiles/SearchEngines.php
-     *
-     * @group Plugins
-     */
-    public function testObsoleteSearchEngineIcons()
-    {
-        include PIWIK_PATH_TEST_TO_ROOT . '/core/DataFiles/SearchEngines.php';
-
-        // Get list of search engines and first appearing URL
-        $searchEngines = array();
-        foreach ($GLOBALS['Piwik_SearchEngines'] as $url => $searchEngine) {
-            $name = parse_url('http://' . $url);
-            if (!array_key_exists($name['host'], $searchEngines)) {
-                $searchEngines[$name['host']] = true;
-            }
-        }
-
-        // Get list of existing favicons
-        $favicons = scandir(PIWIK_PATH_TEST_TO_ROOT . '/plugins/Referrers/images/searchEngines/');
-        foreach ($favicons as $name) {
-            if ($name[0] == '.' || strpos($name, 'xx.') === 0) {
-                continue;
-            }
-
-            $host = substr($name, 0, -4);
-            $this->assertTrue(array_key_exists($host, $searchEngines), $host);
-        }
-    }
-
-    /**
-     * get search engine host from url
-     *
-     * @group Plugins
-     */
-    public function testGetSearchEngineHostFromUrl()
-    {
-        $data = array(
-            'http://www.google.com/cse' => array('www.google.com', 'www.google.com/cse'),
-            'http://www.google.com'     => array('www.google.com', 'www.google.com'),
-        );
-
-        foreach ($data as $url => $expected) {
-            $this->assertEquals($expected[0], \Piwik\Plugins\Referrers\getSearchEngineHostFromUrl($url));
-            $this->assertEquals($expected[1], \Piwik\Plugins\Referrers\getSearchEngineHostPathFromUrl($url));
-        }
-    }
-
-    /**
-     * Dataprovider for testGetSearchEngineUrlFromUrlAndKeyword
-     */
-    public function getSearchEngineUrlFromUrlAndKeywordTestData()
-    {
-        return array(
-            array('http://apollo.lv/portal/search/', 'piwik', 'http://apollo.lv/portal/search/?cof=FORID%3A11&q=piwik&search_where=www'),
-            array('http://bing.com/images/search', 'piwik', 'http://bing.com/images/search/?q=piwik'),
-            array('http://google.com', 'piwik', 'http://google.com/search?q=piwik'),
-        );
-    }
-
-    /**
-     * get search engine url from name and keyword
-     *
-     * @group Plugins
-     *
-     * @dataProvider getSearchEngineUrlFromUrlAndKeywordTestData
-     */
-    public function testGetSearchEngineUrlFromUrlAndKeyword($url, $keyword, $expected)
-    {
-        include PIWIK_PATH_TEST_TO_ROOT . '/core/DataFiles/SearchEngines.php';
-        $this->assertEquals($expected, \Piwik\Plugins\Referrers\getSearchEngineUrlFromUrlAndKeyword($url, $keyword));
-    }
-
-    /**
-     * Dataprovider for getSocialNetworkFromDomainTestData
-     */
-    public function getSocialNetworkFromDomainTestData()
-    {
-        return array(
-            array('http://www.facebook.com', 'Facebook'),
-            array('http://www.facebook.com/piwik', 'Facebook'),
-            array('http://m.facebook.com', 'Facebook'),
-            array('https://m.facebook.com', 'Facebook'),
-            array('m.facebook.com', 'Facebook'),
-            array('http://lastfm.com.tr', 'Last.fm'),
-            array('http://t.co/test', 'Twitter'),
-            array('http://xxt.co/test', \Piwik\Piwik::translate('General_Unknown')),
-            array('asdfasdfadsf.com', \Piwik\Piwik::translate('General_Unknown')),
-            array('http://xwayn.com', \Piwik\Piwik::translate('General_Unknown')),
-            array('http://live.com/test', \Piwik\Piwik::translate('General_Unknown')),
-        );
-    }
-
-    /**
-     * @group Plugins
-     *
-     * @dataProvider getSocialNetworkFromDomainTestData
-     */
-    public function testGetSocialNetworkFromDomain($url, $expected)
-    {
-        include PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php';
-        $this->assertEquals($expected, \Piwik\Plugins\Referrers\getSocialNetworkFromDomain($url));
-    }
-
-    public function getSocialsLogoFromUrlTestData()
-    {
-        return array(
-            array('http://www.facebook.com', 'facebook.com.png'),
-            array('www.facebook.com', 'facebook.com.png',),
-            array('http://lastfm.com.tr', 'last.fm.png'),
-            array('http://asdfasdf.org/test', 'xx.png'),
-            array('http://www.google.com', 'xx.png'),
-        );
-    }
-
-    /**
-     * @group Plugins
-     *
-     * @dataProvider getSocialsLogoFromUrlTestData
-     */
-    public function testGetSocialsLogoFromUrl($url, $expected)
-    {
-        include PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php';
-        $this->assertContains($expected, \Piwik\Plugins\Referrers\getSocialsLogoFromUrl($url));
-    }
-
-    public function isSocialUrlTestData()
-    {
-        return array(
-            array('http://www.facebook.com', 'Facebook', true),
-            array('http://www.facebook.com', 'Twitter', false),
-            array('http://m.facebook.com', false, true),
-            array('http://lastfm.com.tr', 'Last.fm', true),
-            array('http://asdfasdf.org/test', false, false),
-            array('http://asdfasdf.com/test', 'Facebook', false),
-        );
-    }
-
-    /**
-     * @group Plugins
-     *
-     * @dataProvider isSocialUrlTestData
-     */
-    public function testIsSocialUrl($url, $assumedSocial, $expected)
-    {
-        include PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php';
-        $this->assertEquals($expected, \Piwik\Plugins\Referrers\isSocialUrl($url, $assumedSocial));
-    }
-
     public function removeUrlProtocolTestData()
     {
         return array(
diff --git a/plugins/Referrers/tests/Unit/SearchEngineTest.php b/plugins/Referrers/tests/Unit/SearchEngineTest.php
new file mode 100644
index 0000000000..e9f0c926ea
--- /dev/null
+++ b/plugins/Referrers/tests/Unit/SearchEngineTest.php
@@ -0,0 +1,173 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\Referrers\tests;
+
+use Piwik\Plugins\Referrers\SearchEngine;
+use Spyc;
+
+/**
+ * @group SearchEngine
+ */
+class SearchEngineTest extends \PHPUnit_Framework_TestCase
+{
+    public static function setUpBeforeClass()
+    {
+        // inject definitions to avoid database usage
+        $yml = file_get_contents(PIWIK_PATH_TEST_TO_ROOT . SearchEngine::DEFINITION_FILE);
+        SearchEngine::getInstance()->loadYmlData($yml);
+        parent::setUpBeforeClass();
+    }
+
+    public function getSearchEngineUrls()
+    {
+        return Spyc::YAMLLoad(PIWIK_PATH_TEST_TO_ROOT .'/tests/resources/extractSearchEngineInformationFromUrlTests.yml');
+    }
+
+    /**
+     * @dataProvider getSearchEngineUrls
+     */
+    public function testExtractInformationFromUrl($url, $engine, $keywords)
+    {
+        $returnedValue = SearchEngine::getInstance()->extractInformationFromUrl($url);
+
+        $expectedValue = false;
+
+        if (!empty($engine)) {
+            $expectedValue = array('name' => $engine, 'keywords' => $keywords);
+        }
+
+        $this->assertEquals($expectedValue, $returnedValue);
+    }
+
+    public function testSearchEnginesDefinedCorrectly()
+    {
+        $searchEngines = array();
+        foreach (SearchEngine::getInstance()->getDefinitions() as $host => $info) {
+            if (isset($info['backlink']) && $info['backlink'] !== false) {
+                $this->assertTrue(strrpos($info['backlink'], "{k}") !== false, $host . " search URL is not defined correctly, must contain the macro {k}");
+            }
+
+            if (!array_key_exists($info['name'], $searchEngines)) {
+                $searchEngines[$info['name']] = true;
+
+                $this->assertTrue(strpos($host, '{}') === false, $host . " search URL is the master record and should not contain {}");
+            }
+
+            if (isset($info['charsets']) && $info['charsets'] !== false) {
+                $this->assertTrue(is_array($info['charsets']) || is_string($info['charsets']), $host . ' charsets must be either a string or an array');
+
+                if (is_string($info['charsets'])) {
+                    $this->assertTrue(trim($info['charsets']) !== '', $host . ' charsets cannot be an empty string');
+                    $this->assertTrue(strpos($info['charsets'], ' ') === false, $host . ' charsets cannot contain spaces');
+
+                }
+
+                if (is_array($info['charsets'])) {
+                    $this->assertTrue(count($info['charsets']) > 0, $host . ' charsets cannot be an empty array');
+                    $this->assertTrue(strpos(serialize($info['charsets']), '""') === false, $host . ' charsets in array cannot be empty stringss');
+                    $this->assertTrue(strpos(serialize($info['charsets']), ' ') === false, $host . ' charsets in array cannot contain spaces');
+                }
+            }
+        }
+    }
+
+    /**
+     * Dataprovider for testGetBackLinkFromUrlAndKeyword
+     */
+    public function getBackLinkFromUrlAndKeywordTestData()
+    {
+        return array(
+            array('http://apollo.lv/portal/search/', 'piwik', 'http://apollo.lv/portal/search/?cof=FORID%3A11&q=piwik&search_where=www'),
+            array('http://bing.com/images/search', 'piwik', 'http://bing.com/images/search/?q=piwik'),
+            array('http://google.com', 'piwik', 'http://google.com/search?q=piwik'),
+        );
+    }
+
+    /**
+     * get search engine url from name and keyword
+     *
+     * @dataProvider getBackLinkFromUrlAndKeywordTestData
+     */
+    public function testGetBackLinkFromUrlAndKeyword($url, $keyword, $expected)
+    {
+        $this->assertEquals($expected, SearchEngine::getInstance()->getBackLinkFromUrlAndKeyword($url, $keyword));
+    }
+
+    /**
+     * Dataprovider serving all search engine data
+     */
+    public function getAllSearchEngines()
+    {
+        $yml = file_get_contents(PIWIK_PATH_TEST_TO_ROOT . SearchEngine::DEFINITION_FILE);
+        SearchEngine::getInstance()->loadYmlData($yml);
+        $searchEngines = array();
+        foreach (SearchEngine::getInstance()->getDefinitions() as $url => $searchEngine) {
+            $searchEngines[] = array($url, $searchEngine);
+        }
+        return $searchEngines;
+    }
+
+    /**
+     * search engine has at least one keyword
+     *
+     * @dataProvider getAllSearchEngines
+     */
+    public function testMissingSearchEngineKeyword($url, $searchEngine)
+    {
+        $name = parse_url('http://' . $url);
+        $this->assertTrue(!empty($searchEngine['params']), $name['host']);
+    }
+
+    /**
+     * search engine is defined but there's no favicon
+     *
+     * @dataProvider getAllSearchEngines
+     */
+    public function testMissingSearchEngineIcons($url, $searchEngine)
+    {
+        // Get list of existing favicons
+        $favicons = scandir(PIWIK_PATH_TEST_TO_ROOT . '/plugins/Referrers/images/searchEngines/');
+
+        // Get list of search engines and first appearing URL
+        static $searchEngines = array();
+
+        $name = parse_url('http://' . $url);
+        if (!array_key_exists($searchEngine['name'], $searchEngines)) {
+            $searchEngines[$searchEngine['name']] = $url;
+
+            $this->assertTrue(in_array($name['host'] . '.png', $favicons), $name['host']);
+        }
+    }
+
+    /**
+     * favicon exists but there's no corresponding search engine defined
+     */
+    public function testObsoleteSearchEngineIcons()
+    {
+        // Get list of search engines and first appearing URL
+        $searchEngines = array();
+        foreach (SearchEngine::getInstance()->getDefinitions() as $url => $searchEngine) {
+            $name = parse_url('http://' . $url);
+            if (!array_key_exists($name['host'], $searchEngines)) {
+                $searchEngines[$name['host']] = true;
+            }
+        }
+
+        // Get list of existing favicons
+        $favicons = scandir(PIWIK_PATH_TEST_TO_ROOT . '/plugins/Referrers/images/searchEngines/');
+        foreach ($favicons as $name) {
+            if ($name[0] == '.' || strpos($name, 'xx.') === 0) {
+                continue;
+            }
+
+            $host = substr($name, 0, -4);
+            $this->assertTrue(array_key_exists($host, $searchEngines), $host);
+        }
+    }
+}
\ No newline at end of file
diff --git a/plugins/Referrers/tests/Unit/SocialTest.php b/plugins/Referrers/tests/Unit/SocialTest.php
new file mode 100644
index 0000000000..8e8f35e0f9
--- /dev/null
+++ b/plugins/Referrers/tests/Unit/SocialTest.php
@@ -0,0 +1,97 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\Referrers\tests;
+
+use Piwik\Plugins\Referrers\Social;
+use Spyc;
+
+/**
+ * @group Social
+ * @group Plugins
+ */
+class SocialTest extends \PHPUnit_Framework_TestCase
+{
+    public static function setUpBeforeClass()
+    {
+        // inject definitions to avoid database usage
+        $yml = file_get_contents(PIWIK_PATH_TEST_TO_ROOT . Social::DEFINITION_FILE);
+        Social::getInstance()->loadYmlData($yml);
+        parent::setUpBeforeClass();
+    }
+
+    public function isSocialUrlTestData()
+    {
+        return array(
+            array('http://www.facebook.com', 'Facebook', true),
+            array('http://www.facebook.com', 'Twitter', false),
+            array('http://m.facebook.com', false, true),
+            array('http://lastfm.com.tr', 'Last.fm', true),
+            array('http://asdfasdf.org/test', false, false),
+            array('http://asdfasdf.com/test', 'Facebook', false),
+        );
+    }
+
+    /**
+     * @dataProvider isSocialUrlTestData
+     */
+    public function testIsSocialUrl($url, $assumedSocial, $expected)
+    {
+        $this->assertEquals($expected, Social::getInstance()->isSocialUrl($url, $assumedSocial));
+    }
+
+
+    /**
+     * Dataprovider for getSocialNetworkFromDomainTestData
+     */
+    public function getSocialNetworkFromDomainTestData()
+    {
+        return array(
+            array('http://www.facebook.com', 'Facebook'),
+            array('http://www.facebook.com/piwik', 'Facebook'),
+            array('http://m.facebook.com', 'Facebook'),
+            array('https://m.facebook.com', 'Facebook'),
+            array('m.facebook.com', 'Facebook'),
+            array('http://lastfm.com.tr', 'Last.fm'),
+            array('http://t.co/test', 'Twitter'),
+            array('http://xxt.co/test', \Piwik\Piwik::translate('General_Unknown')),
+            array('asdfasdfadsf.com', \Piwik\Piwik::translate('General_Unknown')),
+            array('http://xwayn.com', \Piwik\Piwik::translate('General_Unknown')),
+            array('http://live.com/test', \Piwik\Piwik::translate('General_Unknown')),
+        );
+    }
+
+    /**
+     * @dataProvider getSocialNetworkFromDomainTestData
+     */
+    public function testGetSocialNetworkFromDomain($url, $expected)
+    {
+        $this->assertEquals($expected, Social::getInstance()->getSocialNetworkFromDomain($url));
+    }
+
+    public function getLogoFromUrlTestData()
+    {
+        return array(
+            array('http://www.facebook.com', 'facebook.com.png'),
+            array('www.facebook.com', 'facebook.com.png',),
+            array('http://lastfm.com.tr', 'last.fm.png'),
+            array('http://asdfasdf.org/test', 'xx.png'),
+            array('http://www.google.com', 'xx.png'),
+        );
+    }
+
+    /**
+     * @group Plugins
+     *
+     * @dataProvider getLogoFromUrlTestData
+     */
+    public function testGetLogoFromUrl($url, $expected)
+    {
+        $this->assertContains($expected, Social::getInstance()->getLogoFromUrl($url));
+    }
+}
\ No newline at end of file