Merge pull request #9320 from piwik/website_path

Added support to specify a path in a website URL and Piwik will consider the path during tracking

11 files changed, 496 insertions, 22 deletions

diff --git a/plugins/CoreAdminHome/javascripts/jsTrackingGenerator.js b/plugins/CoreAdminHome/javascripts/jsTrackingGenerator.js
index 87b053e980..3c983deca8 100644
--- a/plugins/CoreAdminHome/javascripts/jsTrackingGenerator.js
+++ b/plugins/CoreAdminHome/javascripts/jsTrackingGenerator.js
@@ -302,7 +302,7 @@
         $('#javascript-text>textarea,#image-tracking-text>textarea').click(function () {
             $(this).select();
         });
-
+        
         // initial generation
         getSiteData(
             $('#js-tracker-website').attr('siteid'),
diff --git a/plugins/CoreAdminHome/templates/trackingCodeGenerator.twig b/plugins/CoreAdminHome/templates/trackingCodeGenerator.twig
index 25d0352733..6d66695552 100644
--- a/plugins/CoreAdminHome/templates/trackingCodeGenerator.twig
+++ b/plugins/CoreAdminHome/templates/trackingCodeGenerator.twig
@@ -75,7 +75,7 @@
                 {{ 'CoreAdminHome_JSTracking_MergeAliasesDesc'|translate("<span class='current-site-alias'>"~defaultReportSiteAlias~"</span>")|raw }}
             </div>
             <label class="checkbox">
-                <input type="checkbox" id="javascript-tracking-all-aliases"/>
+                <input type="checkbox" checked="checked" id="javascript-tracking-all-aliases"/>
                 {{ 'CoreAdminHome_JSTracking_MergeAliases'|translate }}
                 <span class='current-site-name'>{{ defaultReportSiteName|raw }}</span>
             </label>
diff --git a/plugins/Referrers/Columns/Base.php b/plugins/Referrers/Columns/Base.php
index 900c7bba63..a7e7eaf1e3 100644
--- a/plugins/Referrers/Columns/Base.php
+++ b/plugins/Referrers/Columns/Base.php
@@ -12,6 +12,8 @@ use Piwik\Common;
 use Piwik\Piwik;
 use Piwik\Plugin\Dimension\VisitDimension;
 use Piwik\Plugins\Referrers\SearchEngine AS SearchEngineDetection;
+use Piwik\Plugins\SitesManager\SiteUrls;
+use Piwik\Tracker\Cache;
 use Piwik\Tracker\PageUrl;
 use Piwik\Tracker\Request;
 use Piwik\Tracker\Visit;
@@ -251,20 +253,34 @@ abstract class Base extends VisitDimension
      */
     protected function detectReferrerDirectEntry()
     {
-        if (!empty($this->referrerHost)) {
-            // is the referrer host the current host?
-            if (isset($this->currentUrlParse['host'])) {
-                $currentHost = Common::mb_strtolower($this->currentUrlParse['host'], 'UTF-8');
-                if ($currentHost == Common::mb_strtolower($this->referrerHost, 'UTF-8')) {
-                    $this->typeReferrerAnalyzed = Common::REFERRER_TYPE_DIRECT_ENTRY;
-                    return true;
-                }
+        if (empty($this->referrerHost)) {
+            return false;
+        }
+
+        $cache = Cache::getCacheGeneral();
+
+        if (!empty($cache['allUrlsByHostAndIdSite'])) {
+            $directEntry   = new SiteUrls();
+            $matchingSites = $directEntry->getIdSitesMatchingUrl($this->referrerUrlParse, $cache['allUrlsByHostAndIdSite']);
+
+            if (isset($matchingSites) && is_array($matchingSites) && in_array($this->idsite, $matchingSites)) {
+                $this->typeReferrerAnalyzed = Common::REFERRER_TYPE_DIRECT_ENTRY;
+                return true;
+            } elseif (isset($matchingSites)) {
+                return false;
             }
-            if (Visit::isHostKnownAliasHost($this->referrerHost, $this->idsite)) {
+        }
+
+        // fallback logic if the referrer domain is not known to any site to not break BC
+        if (isset($this->currentUrlParse['host'])) {
+            // this might be actually buggy if first thing tracked is eg an outlink and referrer is from that site
+            $currentHost = Common::mb_strtolower($this->currentUrlParse['host']);
+            if ($currentHost == Common::mb_strtolower($this->referrerHost)) {
                 $this->typeReferrerAnalyzed = Common::REFERRER_TYPE_DIRECT_ENTRY;
                 return true;
             }
         }
+
         return false;
     }
 
diff --git a/plugins/Referrers/Columns/Campaign.php b/plugins/Referrers/Columns/Campaign.php
index ff2d5c2401..c29b622336 100644
--- a/plugins/Referrers/Columns/Campaign.php
+++ b/plugins/Referrers/Columns/Campaign.php
@@ -37,7 +37,7 @@ class Campaign extends Base
 
     /**
      * If we should create a new visit when the campaign changes, check if the campaign info changed and if so
-     * force the tracker to create a new visit.
+     * force the tracker to create a new visit.i
      *
      * @param Request $request
      * @param Visitor $visitor
diff --git a/plugins/Referrers/Referrers.php b/plugins/Referrers/Referrers.php
index ce8667929b..d262c79224 100644
--- a/plugins/Referrers/Referrers.php
+++ b/plugins/Referrers/Referrers.php
@@ -12,6 +12,7 @@ use Piwik\ArchiveProcessor;
 use Piwik\Common;
 use Piwik\Piwik;
 use Piwik\Plugins\CoreVisualizations\Visualizations\HtmlTable;
+use Piwik\Plugins\SitesManager\SiteUrls;
 
 /**
  * @see plugins/Referrers/functions.php
@@ -31,9 +32,18 @@ class Referrers extends \Piwik\Plugin
             'Insights.addReportToOverview'      => 'addReportToInsightsOverview',
             'Live.getAllVisitorDetails'         => 'extendVisitorDetails',
             'Request.getRenamedModuleAndAction' => 'renameDeprecatedModuleAndAction',
+            'Tracker.setTrackerCacheGeneral'    => 'setTrackerCacheGeneral'
         );
     }
 
+    public function setTrackerCacheGeneral(&$cacheContent)
+    {
+        $siteUrls = new SiteUrls();
+        $urls = $siteUrls->getAllCachedSiteUrls();
+
+        return $cacheContent['allUrlsByHostAndIdSite'] = $siteUrls->groupUrlsByHost($urls);
+    }
+
     public function renameDeprecatedModuleAndAction(&$module, &$action)
     {
         if($module == 'Referers') {
diff --git a/plugins/Referrers/tests/Integration/Columns/ReferrerTypeTest.php b/plugins/Referrers/tests/Integration/Columns/ReferrerTypeTest.php
new file mode 100644
index 0000000000..0d5e0a68ca
--- /dev/null
+++ b/plugins/Referrers/tests/Integration/Columns/ReferrerTypeTest.php
@@ -0,0 +1,127 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\Referrers\tests\Integration\Columns;
+
+use Piwik\Common;
+use Piwik\Plugins\Referrers\Columns\ReferrerType;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+use Piwik\Tracker\Cache;
+use Piwik\Tracker\Request;
+use Piwik\Tracker\Visit\VisitProperties;
+use Piwik\Tracker\Visitor;
+
+/**
+ * @group Referrers
+ * @group ReferrerTypeTest
+ * @group Plugins
+ */
+class ReferrerTypeTest extends IntegrationTestCase
+{
+    /**
+     * @var ReferrerType
+     */
+    private $referrerType;
+    private $idSite1 = 1;
+    private $idSite2 = 2;
+    private $idSite3 = 3;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        Cache::clearCacheGeneral();
+
+        $date = '2012-01-01 00:00:00';
+        $ecommerce = false;
+
+        Fixture::createWebsite($date, $ecommerce, $name = 'test1', $url = 'http://piwik.org/foo/bar');
+        Fixture::createWebsite($date, $ecommerce, $name = 'test2', $url = 'http://piwik.org/');
+        Fixture::createWebsite($date, $ecommerce, $name = 'test3', $url = 'http://piwik.pro/');
+
+        $this->referrerType = new ReferrerType();
+    }
+
+    public function tearDown()
+    {
+        // clean up your test here if needed
+        Cache::clearCacheGeneral();
+
+        parent::tearDown();
+    }
+
+    /**
+     * @dataProvider getReferrerUrls
+     */
+    public function test_onNewVisit_shouldDetectCorrectReferrerType($expectedType, $idSite, $url, $referrerUrl)
+    {
+        $request = $this->getRequest(array('idsite' => $idSite, 'url' => $url, 'urlref' => $referrerUrl));
+        $type = $this->referrerType->onNewVisit($request, $this->getNewVisitor(), $action = null);
+
+        $this->assertSame($expectedType, $type);
+    }
+
+    public function getReferrerUrls()
+    {
+        $url = 'http://piwik.org/foo/bar';
+        $referrer = 'http://piwik.org';
+
+        // $expectedType,                             $idSite,        $url, $referrerUrl
+        return array(
+            // domain matches but path does not match for idsite1
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite1, $url, $referrer),
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite1, $url, $referrer . '/'),
+            // idSite2 matches any piwik.org path so this is a direct entry for it
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite2, $url, $referrer),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite2, $url, $referrer . '/'),
+            // idSite3 has different domain so it is coming from different website
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite3, $url, $referrer),
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite3, $url, $referrer . '/'),
+
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite1, $url, $referrer . '/foo/bar/baz'),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite1, $url, $referrer . '/foo/bar/baz/'),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite1, $url, $referrer . '/foo/bar/baz?x=5'),
+            // /not/xyz belongs to different website
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite1, $url, $referrer . '/not/xyz'),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite2, $url, $referrer . '/not/xyz'),
+
+            // /foo/bar/baz belongs to different website
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite2, $url, $referrer . '/foo/bar/baz'),
+
+            // website as it is from different domain anyway
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite3, $url, $referrer . '/foo/bar/baz'),
+
+            // should detect campaign independent of domain / path
+            array(Common::REFERRER_TYPE_CAMPAIGN,     $this->idSite1, $url . '?pk_campaign=test', $referrer),
+            array(Common::REFERRER_TYPE_CAMPAIGN,     $this->idSite2, $url . '?pk_campaign=test', $referrer),
+            array(Common::REFERRER_TYPE_CAMPAIGN,     $this->idSite3, $url . '?pk_campaign=test', $referrer),
+
+            array(Common::REFERRER_TYPE_SEARCH_ENGINE, $this->idSite3, $url, 'http://google.com/search?q=piwik'),
+
+            // testing case for backwards compatibility where url has same domain as urlref but the domain is not known to any website
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite3, 'http://example.com/foo', 'http://example.com/bar'),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite3, 'http://example.com/foo', 'http://example.com'),
+            array(Common::REFERRER_TYPE_DIRECT_ENTRY, $this->idSite3, 'http://example.com', 'http://example.com/bar'),
+
+            // testing case where domain of referrer is not known to any site but neither is the URL, url != urlref
+            array(Common::REFERRER_TYPE_WEBSITE,      $this->idSite3, 'http://example.org', 'http://example.com/bar'),
+        );
+    }
+
+    private function getRequest($params)
+    {
+        return new Request($params);
+    }
+
+    private function getNewVisitor()
+    {
+        return new Visitor(new VisitProperties());
+    }
+
+}
diff --git a/plugins/SitesManager/API.php b/plugins/SitesManager/API.php
index 9a6bf63469..739af7d5e9 100644
--- a/plugins/SitesManager/API.php
+++ b/plugins/SitesManager/API.php
@@ -645,6 +645,7 @@ class API extends \Piwik\Plugin\API
     {
         Site::clearCache();
         Cache::regenerateCacheWebsiteAttributes($idSite);
+        Cache::clearCacheGeneral();
         SiteUrls::clearSitesCache();
     }
 
diff --git a/plugins/SitesManager/Model.php b/plugins/SitesManager/Model.php
index 93ddb01d8f..96062b608a 100644
--- a/plugins/SitesManager/Model.php
+++ b/plugins/SitesManager/Model.php
@@ -286,6 +286,22 @@ class Model
         return $urls;
     }
 
+    /**
+     * Returns the list of alias URLs registered for the given idSite.
+     * The website ID must be valid when calling this method!
+     *
+     * @param int $idSite
+     * @return array list of alias URLs
+     */
+    public function getAllKnownUrlsForAllSites()
+    {
+        $db        = $this->getDb();
+        $mainUrls  = $db->fetchAll("SELECT idsite, main_url as url FROM " . Common::prefixTable("site"));
+        $aliasUrls = $db->fetchAll("SELECT idsite, url FROM " . Common::prefixTable("site_url"));
+
+        return array_merge($mainUrls, $aliasUrls);
+    }
+
     public function updateSite($site, $idSite)
     {
         $idSite = (int) $idSite;
diff --git a/plugins/SitesManager/SiteUrls.php b/plugins/SitesManager/SiteUrls.php
index a1fac67a89..059521a4a9 100644
--- a/plugins/SitesManager/SiteUrls.php
+++ b/plugins/SitesManager/SiteUrls.php
@@ -9,6 +9,7 @@
 namespace Piwik\Plugins\SitesManager;
 
 use Piwik\Cache;
+use Piwik\Common;
 
 class SiteUrls
 {
@@ -19,6 +20,96 @@ class SiteUrls
         self::getCache()->delete(self::$cacheId);
     }
 
+    /**
+     * Groups all URLs by host, path and idsite.
+     *
+     * @param array $urls  An array containing URLs by idsite,
+     *                     eg array(array($idSite = 1 => array('apache.piwik', 'apache2.piwik'), 2 => array(), ...))
+     *                     as returned by {@link getAllCachedSiteUrls()} and {@link getAllSiteUrls}
+     * @return array All urls grouped by host => path => idSites. Path having the most '/' will be listed first
+     * array(
+         'apache.piwik' => array(
+             '/test/two' => $idsite = array(3),
+             '/test' => $idsite = array(1),
+             '/' => $idsite = array(2),
+         ),
+         'test.apache.piwik' => array(
+             '/test/two' => $idsite = array(3),
+             '/test' => $idsite = array(1),
+             '/' => $idsite = array(2, 3),
+         ),
+       );
+     */
+    public function groupUrlsByHost($siteUrls)
+    {
+        if (empty($siteUrls)) {
+            return array();
+        }
+
+        $allUrls = array();
+
+        foreach ($siteUrls as $idSite => $urls) {
+            $idSite = (int) $idSite;
+            foreach ($urls as $url) {
+                $urlParsed = @parse_url($url);
+
+                if ($urlParsed === false || !isset($urlParsed['host'])) {
+                    continue;
+                }
+
+                $host = $this->toCanonicalHost($urlParsed['host']);
+                $path = $this->getCanonicalPathFromParsedUrl($urlParsed);
+
+                if (!isset($allUrls[$host])) {
+                    $allUrls[$host] = array();
+                }
+
+                if (!isset($allUrls[$host][$path])) {
+                    $allUrls[$host][$path] = array();
+                }
+
+                if (!in_array($idSite, $allUrls[$host][$path])) {
+                    $allUrls[$host][$path][] = $idSite;
+                }
+            }
+        }
+
+        foreach ($allUrls as $host => $paths) {
+            uksort($paths, array($this, 'sortByPathDepth'));
+            $allUrls[$host] = $paths;
+        }
+
+        return $allUrls;
+    }
+
+    public function getIdSitesMatchingUrl($parsedUrl, $urlsGroupedByHost)
+    {
+        if (empty($parsedUrl['host'])) {
+            return null;
+        }
+
+        $urlHost = $this->toCanonicalHost($parsedUrl['host']);
+        $urlPath = $this->getCanonicalPathFromParsedUrl($parsedUrl);
+
+        $matchingSites = null;
+        if (isset($urlsGroupedByHost[$urlHost])) {
+            $paths = $urlsGroupedByHost[$urlHost];
+
+            foreach ($paths as $path => $idSites) {
+                if (0 === strpos($urlPath, $path)) {
+                    $matchingSites = $idSites;
+                    break;
+                }
+            }
+
+            if (!isset($matchingSites) && isset($paths['/'])) {
+                $matchingSites = $paths['/'];
+            }
+        }
+
+        return $matchingSites;
+    }
+
     public function getAllCachedSiteUrls()
     {
         $cache    = $this->getCache();
@@ -35,23 +126,66 @@ class SiteUrls
     public function getAllSiteUrls()
     {
         $model    = new Model();
-        $siteIds  = $model->getSitesId();
-        $siteUrls = array();
+        $siteUrls = $model->getAllKnownUrlsForAllSites();
 
-        if (empty($siteIds)) {
+        if (empty($siteUrls)) {
             return array();
         }
 
-        foreach ($siteIds as $siteId) {
-            $siteId = (int) $siteId;
-            $siteUrls[$siteId] = $model->getSiteUrlsFromId($siteId);
+        $urls = array();
+        foreach ($siteUrls as $siteUrl) {
+            $siteId = (int) $siteUrl['idsite'];
+
+            if (!isset($urls[$siteId])) {
+                $urls[$siteId] = array();
+            }
+
+            $urls[$siteId][] = $siteUrl['url'];
         }
 
-        return $siteUrls;
+        return $urls;
     }
 
     private static function getCache()
     {
         return Cache::getLazyCache();
     }
+
+    private function sortByPathDepth($pathA, $pathB)
+    {
+        // list first the paths with most '/' , and list path = '/' last
+        $numSlashA = substr_count($pathA, '/');
+        $numSlashB = substr_count($pathB, '/');
+
+        if ($numSlashA === $numSlashB) {
+            return -1 * strcmp($pathA, $pathB);
+        }
+
+        return $numSlashA > $numSlashB ? -1 : 1;
+    }
+
+    private function toCanonicalHost($host)
+    {
+        $host = Common::mb_strtolower($host);
+        if (strpos($host, 'www.') === 0) {
+            $host = substr($host, 4);
+        }
+
+        return $host;
+    }
+
+    private function getCanonicalPathFromParsedUrl($urlParsed)
+    {
+        $path = '/';
+
+        if (isset($urlParsed['path'])) {
+            $path = Common::mb_strtolower($urlParsed['path']);
+            if (!Common::stringEndsWith($path, '/')) {
+                $path .= '/';
+            }
+        }
+
+        return $path;
+    }
+
 }
diff --git a/plugins/SitesManager/tests/Integration/ModelTest.php b/plugins/SitesManager/tests/Integration/ModelTest.php
index 45ebce2be8..7291ac346f 100644
--- a/plugins/SitesManager/tests/Integration/ModelTest.php
+++ b/plugins/SitesManager/tests/Integration/ModelTest.php
@@ -56,10 +56,56 @@ class ModelTest extends IntegrationTestCase
         $this->assertSame(array('website', 'universal', 'mobileapp'), $this->model->getUsedTypeIds());
     }
 
-    private function createMeasurable($type)
+    public function test_getAllKnownUrlsForAllSites_shouldReturnAllUrls()
     {
-        Fixture::createWebsite('2015-01-01 00:00:00',
-            $ecommerce = 0, $siteName = false, $siteUrl = false,
+        $idSite = $this->createMeasurable('website', 'http://apache.piwik');
+        $this->model->insertSiteUrl($idSite, 'http://example.apache.piwik');
+        $this->model->insertSiteUrl($idSite, 'http://example.org');
+
+        $idSite2 = $this->createMeasurable('website');
+        $this->model->insertSiteUrl($idSite2, 'http://example.org');
+        $this->model->insertSiteUrl($idSite2, 'http://example.com');
+
+        $idSite3 = $this->createMeasurable('website', 'http://example.pro');
+
+        $expected = array(
+            array(
+                'idsite' => $idSite,
+                'url' => 'http://apache.piwik'
+            ),
+            array(
+                'idsite' => $idSite2,
+                'url' => 'http://piwik.net'
+            ),
+            array(
+                'idsite' => $idSite3,
+                'url' => 'http://example.pro'
+            ),
+            array(
+                'idsite' => $idSite,
+                'url' => 'http://example.apache.piwik'
+            ),
+            array(
+                'idsite' => $idSite,
+                'url' => 'http://example.org'
+            ),
+            array(
+                'idsite' => $idSite2,
+                'url' => 'http://example.com'
+            ),
+            array(
+                'idsite' => $idSite2,
+                'url' => 'http://example.org'
+            )
+
+        );
+        $this->assertEquals($expected, $this->model->getAllKnownUrlsForAllSites());
+    }
+
+    private function createMeasurable($type, $siteUrl = false)
+    {
+        return Fixture::createWebsite('2015-01-01 00:00:00',
+            $ecommerce = 0, $siteName = false, $siteUrl,
             $siteSearch = 1, $searchKeywordParameters = null,
             $searchCategoryParameters = null, $timezone = null, $type);
     }
diff --git a/plugins/SitesManager/tests/Integration/SiteUrlsTest.php b/plugins/SitesManager/tests/Integration/SiteUrlsTest.php
index f9362ae2f4..39d628197e 100644
--- a/plugins/SitesManager/tests/Integration/SiteUrlsTest.php
+++ b/plugins/SitesManager/tests/Integration/SiteUrlsTest.php
@@ -119,6 +119,130 @@ class SiteUrlsTest extends IntegrationTestCase
         $this->assertEquals($urlsToFake, $actual);
     }
 
+    public function test_groupUrlsByHost_shouldReturnEmptyArray_WhenNoUrlsGiven()
+    {
+        $this->assertSame(array(), $this->siteUrls->groupUrlsByHost(array()));
+        $this->assertSame(array(), $this->siteUrls->groupUrlsByHost(null));
+    }
+
+    public function test_groupUrlsByHost_shouldGroupByHost_WithOneSiteAndDifferentDomains_shouldRemoveWwwAndDefaultToPathSlash()
+    {
+        $idSite = 1;
+        $oneSite = array(
+            $idSite => array(
+                'http://apache.piwik',
+                'http://www.example.com',  // should remove www.
+                'https://example.org',     // should handle https or other protocol
+                'http://apache.piwik/',    // same as initial one but with slash at the end, should not add idsite twice
+                'http://third.www.com'     // should not remove www. in the middle of a domain
+            )
+        );
+
+        $expected = array(
+            'apache.piwik'  => array('/' => array($idSite)),
+            'example.com'   => array('/' => array($idSite)),
+            'example.org'   => array('/' => array($idSite)),
+            'third.www.com' => array('/' => array($idSite)),
+        );
+
+        $this->assertSame($expected, $this->siteUrls->groupUrlsByHost($oneSite));
+    }
+
+    public function test_groupUrlsByHost_shouldGroupByHost_WithDifferentDomainsAndPathsShouldListPathByNumberOfDirectoriesAndConvertToLowerCase()
+    {
+        $idSite = 1;
+        $idSite2 = 2;
+        $idSite3 = 3;
+        $idSite4 = 4;
+        $idSite5 = 5;
+
+        $urls = array(
+            $idSite => array(
+                'http://apache.piwik/test', 'http://apache.piWik', 'http://apache.piwik/foo/bAr/', 'http://apache.piwik/Foo/SECOND'
+            ),
+            $idSite2 => array(
+                'http://apache.piwik/test/', 'http://example.oRg', 'http://apache.piwik/foo/secOnd'
+            ),
+            $idSite3 => array(
+                'http://apache.piwik/', 'http://apache.piwik/third', 'http://exampLe.com', 'http://example.org/foo/test/two'
+            ),
+            $idSite4 => array(),
+            $idSite5 => array('invalidUrl', 'ftp://example.org/'),
+        );
+
+        $expected = array(
+            'apache.piwik' => array(
+                '/foo/second/' => array($idSite, $idSite2),
+                '/foo/bar/' => array($idSite),
+                '/third/' => array($idSite3),
+                '/test/' => array($idSite, $idSite2),
+                '/' => array($idSite, $idSite3)
+            ),
+            'example.org' => array(
+                '/foo/test/two/' => array($idSite3),
+                '/' => array($idSite2, $idSite5)
+            ),
+            'example.com' => array(
+                '/' => array($idSite3)
+            ),
+        );
+
+        $this->assertSame($expected, $this->siteUrls->groupUrlsByHost($urls));
+    }
+
+    /**
+     * @dataProvider getTestIdSitesMatchingUrl
+     */
+    public function test_getIdSitesMatchingUrl($expectedMatchSites, $parsedUrl)
+    {
+        $urlsGroupedByHost = array(
+            'apache.piwik' => array(
+                '/foo/second/' => array(2),
+                '/foo/sec/' => array(4),
+                '/foo/bar/' => array(1),
+                '/third/' => array(3),
+                '/test/' => array(1, 2),
+                '/' => array(1, 3)
+            ),
+            'example.org' => array(
+                '/foo/test/two/' => array(3),
+                '/foo/second/' => array(6),
+                '/' => array(2, 5)
+            ),
+            'example.com' => array(
+                '/' => array(3)
+            ),
+        );
+        $matchedSites = $this->siteUrls->getIdSitesMatchingUrl($parsedUrl, $urlsGroupedByHost);
+
+        $this->assertSame($expectedMatchSites, $matchedSites);
+    }
+
+    public function getTestIdSitesMatchingUrl()
+    {
+        return array(
+            array(array(1,3), array('host' => 'apache.piwik')),
+            array(array(1,3), array('host' => 'apache.piwik', 'path' => '/')),
+            array(array(1,3), array('host' => 'apache.piwik', 'path' => 'nomatch')), // no other URL matches a site so we fall back to domain match
+            array(array(1,3), array('host' => 'apache.piwik', 'path' => '/nomatch')),
+            array(array(2), array('host' => 'apache.piwik', 'path' => '/foo/second')),
+            array(array(2), array('host' => 'apache.piwik', 'path' => '/foo/second/')), // it shouldn't matter if slash is at end or not
+            array(array(2), array('host' => 'apache.piwik', 'path' => '/foo/second/test')), // it should find best match
+            array(array(4), array('host' => 'apache.piwik', 'path' => '/foo/sec/test')), // it should not use /foo/second for these
+            array(array(4), array('host' => 'apache.piwik', 'path' => '/foo/sec/')),
+            array(array(4), array('host' => 'apache.piwik', 'path' => '/foo/sec')),
+            array(array(1,3), array('host' => 'apache.piwik', 'path' => '/foo')),
+            array(array(2,5), array('host' => 'example.org')),
+            array(array(2,5), array('host' => 'example.org', 'path' => '/')),
+            array(array(2,5), array('host' => 'example.org', 'path' => 'any/nonmatching/path')),
+            array(array(6), array('host' => 'example.org', 'path' => '/foo/second')),
+            array(array(6), array('host' => 'example.org', 'path' => '/foo/second/test')),
+            array(array(3), array('host' => 'example.com')),
+            array(null, array('host' => 'example.pro')),
+            array(null, array('host' => 'example.pro', 'path' => '/any')),
+        );
+    }
+
     private function assertSiteUrls($expectedUrls)
     {
         $urls = $this->siteUrls->getAllSiteUrls();