refs #9320 when detecting the referrer name, consider a path of a different website if possible and not only domain

5 files changed, 225 insertions, 10 deletions

diff --git a/plugins/Referrers/Columns/Base.php b/plugins/Referrers/Columns/Base.php
index a7e7eaf1e3..a4da79d389 100644
--- a/plugins/Referrers/Columns/Base.php
+++ b/plugins/Referrers/Columns/Base.php
@@ -114,6 +114,14 @@ abstract class Base extends VisitDimension
         if (!$referrerDetected && !empty($this->referrerHost)) {
             $this->typeReferrerAnalyzed = Common::REFERRER_TYPE_WEBSITE;
             $this->nameReferrerAnalyzed = Common::mb_strtolower($this->referrerHost);
+
+            $urlsByHost = $this->getCachedUrlsByHostAndIdSite();
+
+            $directEntry = new SiteUrls();
+            $path = $directEntry->getPathMatchingUrl($this->referrerUrlParse, $urlsByHost);
+            if (!empty($path) && $path !== '/') {
+                $this->nameReferrerAnalyzed .= rtrim($path, '/');
+            }
         }
 
         $referrerInformation = array(
@@ -244,6 +252,17 @@ abstract class Base extends VisitDimension
         return true;
     }
 
+    private function getCachedUrlsByHostAndIdSite()
+    {
+        $cache = Cache::getCacheGeneral();
+
+        if (!empty($cache['allUrlsByHostAndIdSite'])) {
+            return $cache['allUrlsByHostAndIdSite'];
+        }
+
+        return array();
+    }
+
     /**
      * We have previously tried to detect the campaign variables in the URL
      * so at this stage, if the referrer host is the current host,
@@ -257,18 +276,16 @@ abstract class Base extends VisitDimension
             return false;
         }
 
-        $cache = Cache::getCacheGeneral();
+        $urlsByHost = $this->getCachedUrlsByHostAndIdSite();
 
-        if (!empty($cache['allUrlsByHostAndIdSite'])) {
-            $directEntry   = new SiteUrls();
-            $matchingSites = $directEntry->getIdSitesMatchingUrl($this->referrerUrlParse, $cache['allUrlsByHostAndIdSite']);
+        $directEntry   = new SiteUrls();
+        $matchingSites = $directEntry->getIdSitesMatchingUrl($this->referrerUrlParse, $urlsByHost);
 
-            if (isset($matchingSites) && is_array($matchingSites) && in_array($this->idsite, $matchingSites)) {
-                $this->typeReferrerAnalyzed = Common::REFERRER_TYPE_DIRECT_ENTRY;
-                return true;
-            } elseif (isset($matchingSites)) {
-                return false;
-            }
+        if (isset($matchingSites) && is_array($matchingSites) && in_array($this->idsite, $matchingSites)) {
+            $this->typeReferrerAnalyzed = Common::REFERRER_TYPE_DIRECT_ENTRY;
+            return true;
+        } elseif (isset($matchingSites)) {
+            return false;
         }
 
         // fallback logic if the referrer domain is not known to any site to not break BC
diff --git a/plugins/Referrers/tests/Integration/Columns/ReferrerNameTest.php b/plugins/Referrers/tests/Integration/Columns/ReferrerNameTest.php
new file mode 100644
index 0000000000..db34a25c46
--- /dev/null
+++ b/plugins/Referrers/tests/Integration/Columns/ReferrerNameTest.php
@@ -0,0 +1,130 @@
+<?php
+/**
+ * Piwik - free/libre analytics platform
+ *
+ * @link http://piwik.org
+ * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later
+ */
+
+namespace Piwik\Plugins\Referrers\tests\Integration\Columns;
+
+use Piwik\Common;
+use Piwik\Plugins\Referrers\Columns\ReferrerName;
+use Piwik\Plugins\Referrers\Columns\ReferrerType;
+use Piwik\Tests\Framework\Fixture;
+use Piwik\Tests\Framework\TestCase\IntegrationTestCase;
+use Piwik\Tracker\Cache;
+use Piwik\Tracker\Request;
+use Piwik\Tracker\Visit\VisitProperties;
+use Piwik\Tracker\Visitor;
+
+/**
+ * @group Referrers
+ * @group ReferrerNameTest
+ * @group ReferrerName
+ * @group Plugins
+ */
+class ReferrerNameTest extends IntegrationTestCase
+{
+    /**
+     * @var ReferrerType
+     */
+    private $referrerName;
+    private $idSite1 = 1;
+    private $idSite2 = 2;
+    private $idSite3 = 3;
+
+    public function setUp()
+    {
+        parent::setUp();
+
+        Cache::clearCacheGeneral();
+
+        $date = '2012-01-01 00:00:00';
+        $ecommerce = false;
+
+        Fixture::createWebsite($date, $ecommerce, $name = 'test1', $url = 'http://piwik.org/foo/bar');
+        Fixture::createWebsite($date, $ecommerce, $name = 'test2', $url = 'http://piwik.org/');
+        Fixture::createWebsite($date, $ecommerce, $name = 'test3', $url = 'http://piwik.pro/');
+
+        $this->referrerName = new ReferrerName();
+    }
+
+    public function tearDown()
+    {
+        // clean up your test here if needed
+        Cache::clearCacheGeneral();
+
+        parent::tearDown();
+    }
+
+    /**
+     * @dataProvider getReferrerUrls
+     */
+    public function test_onNewVisit_shouldDetectCorrectReferrerType($expectedType, $idSite, $url, $referrerUrl)
+    {
+        $request = $this->getRequest(array('idsite' => $idSite, 'url' => $url, 'urlref' => $referrerUrl));
+        $type = $this->referrerName->onNewVisit($request, $this->getNewVisitor(), $action = null);
+
+        $this->assertSame($expectedType, $type);
+    }
+
+    public function getReferrerUrls()
+    {
+        $url = 'http://piwik.org/foo/bar';
+        $referrer = 'http://piwik.org';
+
+        $directEntryReferrerName = '';
+
+        return array(
+            // domain matches but path does not match for idsite1
+            array('piwik.org',              $this->idSite1, $url, $referrer),
+            array('piwik.org',              $this->idSite1, $url, $referrer . '/'),
+            // idSite2 matches any piwik.org path so this is a direct entry for it
+            array($directEntryReferrerName, $this->idSite2, $url, $referrer),
+            array($directEntryReferrerName, $this->idSite2, $url, $referrer . '/'),
+            // idSite3 has different domain so it is coming from different website
+            array('piwik.org',              $this->idSite3, $url, $referrer),
+            array('piwik.org',              $this->idSite3, $url, $referrer . '/'),
+
+            array($directEntryReferrerName, $this->idSite1, $url, $referrer . '/foo/bar/baz'),
+            array($directEntryReferrerName, $this->idSite1, $url, $referrer . '/foo/bar/baz/'),
+            array($directEntryReferrerName, $this->idSite1, $url, $referrer . '/foo/bar/baz?x=5'),
+            array($directEntryReferrerName, $this->idSite1, $url, $referrer . '/fOo/BaR/baz?x=5'),
+            // /not/xyz belongs to different website
+            array('piwik.org',              $this->idSite1, $url, $referrer . '/not/xyz'),
+            array($directEntryReferrerName, $this->idSite2, $url, $referrer . '/not/xyz'),
+
+            // /foo/bar/baz belongs to different website
+            array('piwik.org/foo/bar',      $this->idSite2, $url, $referrer . '/foo/bar/baz'),
+            array('piwik.org/foo/bar',      $this->idSite3, $url, $referrer . '/foo/bar'),
+            array('piwik.org/foo/bar',      $this->idSite3, $url, $referrer . '/fOo/BaR'),
+
+            // should detect campaign independent of domain / path
+            array('test',                   $this->idSite1, $url . '?pk_campaign=test', $referrer),
+            array('testfoobar',             $this->idSite2, $url . '?pk_campaign=testfoobar', $referrer),
+            array('test',                   $this->idSite3, $url . '?pk_campaign=test', $referrer),
+
+            array('Google',                 $this->idSite3, $url, 'http://google.com/search?q=piwik'),
+
+            // testing case for backwards compatibility where url has same domain as urlref but the domain is not known to any website
+            array($directEntryReferrerName, $this->idSite3, 'http://example.com/foo', 'http://example.com/bar'),
+            array($directEntryReferrerName, $this->idSite3, 'http://example.com/foo', 'http://example.com'),
+            array($directEntryReferrerName, $this->idSite3, 'http://example.com',     'http://example.com/bar'),
+
+            // testing case where domain of referrer is not known to any site but neither is the URL, url != urlref
+            array('example.com',            $this->idSite3, 'http://example.org',     'http://example.com/bar'),
+        );
+    }
+
+    private function getRequest($params)
+    {
+        return new Request($params);
+    }
+
+    private function getNewVisitor()
+    {
+        return new Visitor(new VisitProperties());
+    }
+
+}
diff --git a/plugins/Referrers/tests/Integration/Columns/ReferrerTypeTest.php b/plugins/Referrers/tests/Integration/Columns/ReferrerTypeTest.php
index 0d5e0a68ca..5925dbc5c0 100644
--- a/plugins/Referrers/tests/Integration/Columns/ReferrerTypeTest.php
+++ b/plugins/Referrers/tests/Integration/Columns/ReferrerTypeTest.php
@@ -20,6 +20,7 @@ use Piwik\Tracker\Visitor;
 /**
  * @group Referrers
  * @group ReferrerTypeTest
+ * @group ReferrerType
  * @group Plugins
  */
 class ReferrerTypeTest extends IntegrationTestCase
diff --git a/plugins/SitesManager/SiteUrls.php b/plugins/SitesManager/SiteUrls.php
index 059521a4a9..33e2e4844a 100644
--- a/plugins/SitesManager/SiteUrls.php
+++ b/plugins/SitesManager/SiteUrls.php
@@ -110,6 +110,27 @@ class SiteUrls
         return $matchingSites;
     }
 
+    public function getPathMatchingUrl($parsedUrl, $urlsGroupedByHost)
+    {
+        if (empty($parsedUrl['host'])) {
+            return null;
+        }
+
+        $urlHost = $this->toCanonicalHost($parsedUrl['host']);
+        $urlPath = $this->getCanonicalPathFromParsedUrl($parsedUrl);
+
+        $matchingSites = null;
+        if (isset($urlsGroupedByHost[$urlHost])) {
+            $paths = $urlsGroupedByHost[$urlHost];
+
+            foreach ($paths as $path => $idSites) {
+                if (0 === strpos($urlPath, $path)) {
+                    return $path;
+                }
+            }
+        }
+    }
+
     public function getAllCachedSiteUrls()
     {
         $cache    = $this->getCache();
diff --git a/plugins/SitesManager/tests/Integration/SiteUrlsTest.php b/plugins/SitesManager/tests/Integration/SiteUrlsTest.php
index 39d628197e..3db34199fb 100644
--- a/plugins/SitesManager/tests/Integration/SiteUrlsTest.php
+++ b/plugins/SitesManager/tests/Integration/SiteUrlsTest.php
@@ -243,6 +243,52 @@ class SiteUrlsTest extends IntegrationTestCase
         );
     }
 
+    /**
+     * @dataProvider getTestPathMatchingUrl
+     */
+    public function test_getPathMatchingUrl($expectedMatchSites, $parsedUrl)
+    {
+        $urlsGroupedByHost = array(
+            'apache.piwik' => array(
+                '/foo/second/' => array(2),
+                '/foo/sec/' => array(4),
+                '/foo/bar/' => array(1),
+                '/third/' => array(3),
+                '/test/' => array(1, 2),
+                '/' => array(1, 3)
+            ),
+            'example.org' => array(
+                '/foo/test/two/' => array(3),
+                '/foo/second/' => array(6),
+                '/' => array(2, 5)
+            ),
+        );
+        $matchedSites = $this->siteUrls->getPathMatchingUrl($parsedUrl, $urlsGroupedByHost);
+
+        $this->assertSame($expectedMatchSites, $matchedSites);
+    }
+
+    public function getTestPathMatchingUrl()
+    {
+        return array(
+            array('/', array('host' => 'apache.piwik')),
+            array('/', array('host' => 'apache.piwik', 'path' => '/')),
+            array('/', array('host' => 'apache.piwik', 'path' => '')),
+            array(null, array('host' => 'test.piwik')),
+            array(null, array('host' => 'test.apache.piwik')), // we do not match subdomains, only exact domain match
+
+            array('/foo/bar/', array('host' => 'apache.piwik', 'path' => '/foo/bar')),
+            array('/foo/bar/', array('host' => 'apache.piwik', 'path' => '/foo/bar/')),
+            array('/foo/bar/', array('host' => 'apache.piwik', 'path' => '/foo/bar/baz/')),
+            array('/', array('host' => 'apache.piwik', 'path' => '/foo/baz/bar/')),
+            array('/third/', array('host' => 'apache.piwik', 'path' => '/third/bar/baz/')),
+
+            array('/foo/second/', array('host' => 'example.org', 'path' => '/foo/second/')),
+            array('/', array('host' => 'example.org', 'path' => '/foo/secon')),
+            array(null, array('host' => 'example.pro', 'path' => '/foo/second/')),
+        );
+    }
+
     private function assertSiteUrls($expectedUrls)
     {
         $urls = $this->siteUrls->getAllSiteUrls();