diff options
29 files changed, 1068 insertions, 2187 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 077ece7ace..0d8edb1632 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ This is a changelog for Piwik platform developers. All changes for our HTTP API' ### Internal change * When generating a new plugin skeleton via `generate:plugin` command, plugin name must now contain only letters and numbers. * JavaScript Tracker tests no longer require `SQLite`. The existing MySQL configuration for tests is used now. In order to run the tests make sure Piwik is installed and `[database_tests]` is configured in `config/config.ini.php`. + * The definitions for search engine and social network detection have been moved from bundled data files to a separate package (see [https://github.com/piwik/searchengine-and-social-list](https://github.com/piwik/searchengine-and-social-list)). ### New APIs * Add your own SMS/Text provider by creating a new class in the `SMSProvider` directory of your plugin. The class has to extend `Piwik\Plugins\MobileMessaging\SMSProvider` and implement the required methods. diff --git a/composer.json b/composer.json index 42e741f8fd..f6c0141296 100644 --- a/composer.json +++ b/composer.json @@ -54,6 +54,7 @@ "symfony/event-dispatcher": "~2.6.0", "pear/pear_exception": "~1.0.0", "piwik/referrer-spam-blacklist": "~1.0", + "piwik/searchengine-and-social-list": "~1.0", "tecnickcom/tcpdf": "~6.0" }, "require-dev": { diff --git a/composer.lock b/composer.lock index a6ee66b134..2a53020e14 100644 --- a/composer.lock +++ b/composer.lock @@ -959,6 +959,28 @@ "time": "2015-10-07 10:17:59" }, { + "name": "piwik/searchengine-and-social-list", + "version": "1.0", + "source": { + "type": "git", + "url": "https://github.com/piwik/searchengine-and-social-list.git", + "reference": "e2b97a1cd9ed2dde735de49f8ef9afc26b3df80b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/piwik/searchengine-and-social-list/zipball/e2b97a1cd9ed2dde735de49f8ef9afc26b3df80b", + "reference": "e2b97a1cd9ed2dde735de49f8ef9afc26b3df80b", + "shasum": "" + }, + "type": "library", + "notification-url": "https://packagist.org/downloads/", + "license": [ + "Public Domain" + ], + "description": "Search engine and social network definitions used by Piwik", + "time": "2015-11-06 21:32:51" + }, + { "name": "psr/log", "version": "1.0.0", "source": { diff --git a/core/Common.php b/core/Common.php index 7e3296bee1..95d333db9b 100644 --- a/core/Common.php +++ b/core/Common.php @@ -816,86 +816,6 @@ class Common } /** - * Returns list of search engines by URL - * - * @see core/DataFiles/SearchEngines.php - * - * @return array Array of ( URL => array( searchEngineName, keywordParameter, path, charset ) ) - */ - public static function getSearchEngineUrls() - { - $cacheId = 'Common.getSearchEngineUrls'; - $cache = Cache::getTransientCache(); - $searchEngines = $cache->fetch($cacheId); - - if (empty($searchEngines)) { - require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/SearchEngines.php'; - - $searchEngines = $GLOBALS['Piwik_SearchEngines']; - - Piwik::postEvent('Referrer.addSearchEngineUrls', array(&$searchEngines)); - - $cache->save($cacheId, $searchEngines); - } - - return $searchEngines; - } - - /** - * Returns list of search engines by name - * - * @see core/DataFiles/SearchEngines.php - * - * @return array Array of ( searchEngineName => URL ) - */ - public static function getSearchEngineNames() - { - $cacheId = 'Common.getSearchEngineNames'; - $cache = Cache::getTransientCache(); - $nameToUrl = $cache->fetch($cacheId); - - if (empty($nameToUrl)) { - $searchEngines = self::getSearchEngineUrls(); - - $nameToUrl = array(); - foreach ($searchEngines as $url => $info) { - if (!isset($nameToUrl[$info[0]])) { - $nameToUrl[$info[0]] = $url; - } - } - $cache->save($cacheId, $nameToUrl); - } - - return $nameToUrl; - } - - /** - * Returns list of social networks by URL - * - * @see core/DataFiles/Socials.php - * - * @return array Array of ( URL => Social Network Name ) - */ - public static function getSocialUrls() - { - $cacheId = 'Common.getSocialUrls'; - $cache = Cache::getTransientCache(); - $socialUrls = $cache->fetch($cacheId); - - if (empty($socialUrls)) { - require_once PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php'; - - $socialUrls = $GLOBALS['Piwik_socialUrl']; - - Piwik::postEvent('Referrer.addSocialUrls', array(&$socialUrls)); - - $cache->save($cacheId, $socialUrls); - } - - return $socialUrls; - } - - /** * Returns list of provider names * * @see core/DataFiles/Providers.php diff --git a/core/DataFiles/SearchEngines.php b/core/DataFiles/SearchEngines.php deleted file mode 100644 index 52b59713f6..0000000000 --- a/core/DataFiles/SearchEngines.php +++ /dev/null @@ -1,1140 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - */ - -/** - * Search Engine database - * - * ====================================== - * HOW TO ADD A SEARCH ENGINE TO THE LIST - * ====================================== - * If you want to add a new entry, please email us the information + icon at - * hello at piwik.org - * - * See also: http://piwik.org/faq/general/#faq_39 - * - * Detail of a line: - * Url => array( SearchEngineName, KeywordParameter, [path containing the keyword], [charset used by the search engine]) - * - * The main search engine URL has to be at the top of the list for the given - * search Engine. This serves as the master record so additional URLs - * don't have to duplicate all the information, but can override when needed. - * - * The URL, "example.com", will match "example.com", "m.example.com", - * "www.example.com", and "search.example.com". - * - * For region-specific search engines, the URL, "{}.example.com" will match - * any ISO3166-1 alpha2 country code against "{}". Similarly, "example.{}" - * will match against valid country TLDs, but should be used sparingly to - * avoid false positives. - * - * The charset should be an encoding supported by mbstring. If unspecified, - * we'll assume it's UTF-8. - * Reference: http://www.php.net/manual/en/mbstring.encodings.php - * - * You can add new search engines icons by adding the icon in the - * plugins/Referrers/images/searchEngines directory using the format - * 'mainSearchEngineUrl.png'. Example: www.google.com.png - * - * To help Piwik link directly the search engine result page for the keyword, - * specify the third entry in the array using the macro {k} that will - * automatically be replaced by the keyword. - * - * A simple example is: - * 'www.google.com' => array('Google', 'q', 'search?q={k}'), - * - * A more complicated example, with an array of possible variable names, and a custom charset: - * 'www.baidu.com' => array('Baidu', array('wd', 'word', 'kw'), 's?wd={k}', 'gb2312'), - * - * Another example using a regular expression to parse the path for keywords: - * 'infospace.com' => array('InfoSpace', array('/dir1\/(pattern)\/dir2/'), '/dir1/{k}/dir2/stuff/'), - */ -if (!isset($GLOBALS['Piwik_SearchEngines'])) { - $GLOBALS['Piwik_SearchEngines'] = array( - // 1 - '1.cz' => array('1.cz', array('/s\/([^\/]+)/', 'q'), 's/{k}', 'iso-8859-2'), - - // 123people - 'www.123people.com' => array('123people', array('/s\/([^\/]+)/', 'search_term'), 's/{k}'), - '123people.{}' => array('123people'), - - // 360search - 'so.360.cn' => array('360search', 'q', 's?q={k}', array('UTF-8', 'gb2312')), - 'www.so.com' => array('360search', 'q', 's?q={k}', array('UTF-8', 'gb2312')), - - // Abacho - 'www.abacho.de' => array('Abacho', 'q', 'suche?q={k}'), - 'www.abacho.com' => array('Abacho'), - 'www.abacho.co.uk' => array('Abacho'), - 'www.se.abacho.com' => array('Abacho'), - 'www.tr.abacho.com' => array('Abacho'), - 'www.abacho.at' => array('Abacho'), - 'www.abacho.fr' => array('Abacho'), - 'www.abacho.es' => array('Abacho'), - 'www.abacho.ch' => array('Abacho'), - 'www.abacho.it' => array('Abacho'), - - // ABCsøk - 'abcsok.no' => array('ABCsøk', 'q', '?q={k}'), - 'verden.abcsok.no' => array('ABCsøk'), - - // Acoon - 'www.acoon.de' => array('Acoon', 'begriff', 'cgi-bin/search.exe?begriff={k}'), - - // Aguea - 'chercherfr.aguea.com' => array('Aguea', 'q', 's.py?q={k}'), - - // Alexa - 'alexa.com' => array('Alexa', 'q', 'search?q={k}'), - 'search.toolbars.alexa.com' => array('Alexa'), - - // Alice Adsl - 'rechercher.aliceadsl.fr' => array('Alice Adsl', 'qs', 'google.pl?qs={k}'), - - // Allesklar - 'www.allesklar.de' => array('Allesklar', 'words', '?words={k}'), - 'www.allesklar.at' => array('Allesklar'), - 'www.allesklar.ch' => array('Allesklar'), - - // AllTheWeb - 'www.alltheweb.com' => array('AllTheWeb', 'q', 'search?q={k}'), - - // all.by - 'all.by' => array('All.by', 'query', 'cgi-bin/search.cgi?mode=by&query={k}'), - - // Altavista - 'www.altavista.com' => array('AltaVista', 'q', 'web/results?q={k}'), - 'search.altavista.com' => array('AltaVista'), - 'listings.altavista.com' => array('AltaVista'), - 'altavista.de' => array('AltaVista'), - 'altavista.fr' => array('AltaVista'), - '{}.altavista.com' => array('AltaVista'), - 'be-nl.altavista.com' => array('AltaVista'), - 'be-fr.altavista.com' => array('AltaVista'), - - // Apollo Latvia - 'apollo.lv/portal/search/' => array('Apollo lv', 'q', '?cof=FORID%3A11&q={k}&search_where=www'), - - // APOLLO7 - 'apollo7.de' => array('Apollo7', 'query', 'a7db/index.php?query={k}&de_sharelook=true&de_bing=true&de_witch=true&de_google=true&de_yahoo=true&de_lycos=true'), - - // AOL - 'search.aol.com' => array('AOL', array('query', 'q'), 'aol/search?q={k}'), - 'search.aol.it' => array('AOL'), - 'aolsearch.aol.com' => array('AOL'), - 'www.aolrecherche.aol.fr' => array('AOL'), - 'www.aolrecherches.aol.fr' => array('AOL'), - 'www.aolimages.aol.fr' => array('AOL'), - 'aim.search.aol.com' => array('AOL'), - 'www.recherche.aol.fr' => array('AOL'), - 'recherche.aol.fr' => array('AOL'), - 'find.web.aol.com' => array('AOL'), - 'recherche.aol.ca' => array('AOL'), - 'aolsearch.aol.co.uk' => array('AOL'), - 'search.aol.co.uk' => array('AOL'), - 'aolrecherche.aol.fr' => array('AOL'), - 'sucheaol.aol.de' => array('AOL'), - 'suche.aol.de' => array('AOL'), - 'o2suche.aol.de' => array('AOL'), - 'suche.aolsvc.de' => array('AOL'), - 'aolbusqueda.aol.com.mx' => array('AOL'), - 'alicesuche.aol.de' => array('AOL'), - 'alicesuchet.aol.de' => array('AOL'), - 'suchet2.aol.de' => array('AOL'), - 'search.hp.my.aol.com.au' => array('AOL'), - 'search.hp.my.aol.de' => array('AOL'), - 'search.hp.my.aol.it' => array('AOL'), - 'search-intl.netscape.com' => array('AOL'), - 'de.aolsearch.com' => array('AOL', 'q', 'search?q={k}'), - - // Aport - 'sm.aport.ru' => array('Aport', 'r', 'search?r={k}'), - - // arama - 'arama.com' => array('Arama', 'q', 'search.php3?q={k}'), - - // Arcor - 'www.arcor.de' => array('Arcor', 'Keywords', 'content/searchresult.jsp?Keywords={k}'), - - // Arianna (Libero.it) - 'arianna.libero.it' => array('Arianna', 'query', 'search/abin/integrata.cgi?query={k}'), - 'www.arianna.com' => array('Arianna'), - - // Ask (IAC Search & Media) - 'ask.com' => array('Ask', array('ask', 'q', 'searchfor'), 'web?q={k}'), - 'web.ask.com' => array('Ask'), - 'int.ask.com' => array('Ask'), - 'mws.ask.com' => array('Ask'), - 'images.ask.com' => array('Ask'), - 'images.{}.ask.com' => array('Ask'), - 'ask.reference.com' => array('Ask'), - 'www.askkids.com' => array('Ask'), - 'iwon.ask.com' => array('Ask'), - 'www.ask.co.uk' => array('Ask'), - '{}.ask.com' => array('Ask'), - 'www.qbyrd.com' => array('Ask'), - '{}.qbyrd.com' => array('Ask'), - 'www.search-results.com' => array('Ask'), - 'www1.search-results.com' => array('Ask'), - 'int.search-results.com' => array('Ask'), - '{}.search-results.com' => array('Ask'), - 'search.ask.com' => array('Ask'), - '{}.search.ask.com' => array('Ask'), - 'avira-int.ask.com' => array('Ask'), - 'searchqu.com' => array('Ask'), - 'search.tb.ask.com' => array('Ask'), - 'nortonsafe.search.ask.com' => array('Ask'), - 'safesearch.avira.com' => array('Ask'), - 'avira.search.ask.com' => array('Ask'), - 'int.search.tb.ask.com' => array('Ask'), - - // Atlas - 'searchatlas.centrum.cz' => array('Atlas', 'q', '?q={k}'), - - // auone - 'search.auone.jp' => array('auone', 'q', '?q={k}'), - 'sp-search.auone.jp' => array('auone'), - 'sp-image.search.auone.jp' => array('auone Images', 'q', '?q={k}'), - - // Austronaut - 'www2.austronaut.at' => array('Austronaut', 'q'), - 'www1.austronaut.at' => array('Austronaut'), - - // Babylon (Enhanced by Google), - 'search.babylon.com' => array('Babylon', array('q', '/\/web\/(.*)/'), '?q={k}'), - 'searchassist.babylon.com' => array('Babylon'), - - // Baidu - 'www.baidu.com' => array('Baidu', array('wd', 'word', 'kw'), 's?wd={k}', array('UTF-8', 'gb2312')), - 'www1.baidu.com' => array('Baidu'), - 'm.baidu.com' => array('Baidu'), - 'www.baidu.co.th' => array('Baidu'), - 'zhidao.baidu.com' => array('Baidu'), - 'tieba.baidu.com' => array('Baidu'), - 'news.baidu.com' => array('Baidu'), - 'web.gougou.com' => array('Baidu', 'search', 'search?search={k}'), // uses baidu search - - // Biglobe - 'cgi.search.biglobe.ne.jp' => array('Biglobe', 'q', 'cgi-bin/search-st?q={k}'), - 'images.search.biglobe.ne.jp' => array('Biglobe Images', 'q', 'cgi-bin/search-st?q={k}'), - - // Bing - 'bing.com' => array('Bing', array('q', 'Q'), 'search?q={k}'), - '{}.bing.com' => array('Bing'), - 'msnbc.msn.com' => array('Bing'), - 'dizionario.it.msn.com' => array('Bing'), - 'enciclopedia.it.msn.com' => array('Bing'), - - // Bing Cache - 'cc.bingj.com' => array('Bing'), - - // Bing Images - 'bing.com/images/search' => array('Bing Images', array('q', 'Q'), '?q={k}'), - '{}.bing.com/images/search' => array('Bing Images'), - - // blekko - 'blekko.com' => array('blekko', array('q', '/\/ws\/(.*)/'), 'ws/{k}'), - - // Blogdigger - 'www.blogdigger.com' => array('Blogdigger', 'q'), - - // Blogpulse - 'www.blogpulse.com' => array('Blogpulse', 'query', 'search?query={k}'), - - // Bluewin - 'search.bluewin.ch' => array('Bluewin', array('searchTerm', 'q'), 'v2/index.php?q={k}'), - - // canoe.ca - 'web.canoe.ca' => array('Canoe.ca', 'q', 'search?q={k}'), - - // Centrum - 'search.centrum.cz' => array('Centrum', 'q', '?q={k}'), - 'morfeo.centrum.cz' => array('Centrum'), - - // Charter - 'www.charter.net' => array('Charter', 'q', 'search/index.php?q={k}'), - - // Claro Search - 'claro-search.com' => array('Claro Search', 'q', '?q={k}'), - - // Clix (Enhanced by Google) - 'pesquisa.clix.pt' => array('Clix', 'question', 'resultado.html?in=Mundial&question={k}'), - - // Conduit - 'search.conduit.com' => array('Conduit.com', 'q', 'Results.aspx?q={k}'), - 'images.search.conduit.com' => array('Conduit.com'), - - // Comcast - 'search.comcast.net' => array('Comcast', 'q', '?q={k}'), - - // Crawler - 'www.crawler.com' => array('Crawler', 'q', 'search/results1.aspx?q={k}'), - - // Compuserve - 'websearch.cs.com' => array('Compuserve.com (Enhanced by Google)', 'query', 'cs/search?query={k}'), - - // Cuil - 'www.cuil.com' => array('Cuil', 'q', 'search?q={k}'), - - // Daemon search - 'daemon-search.com' => array('Daemon search', 'q', 'explore/web?q={k}'), - 'my.daemon-search.com' => array('Daemon search'), - - // DasOertliche - 'www.dasoertliche.de' => array('DasOertliche', 'kw'), - 'www2.dasoertliche.de' => array('DasOertliche', array('ph', 'kw')), - - // DasTelefonbuch - 'www1.dastelefonbuch.de' => array('DasTelefonbuch', 'kw'), - - // Daum - 'search.daum.net' => array('Daum', 'q', 'search?q={k}'), - - // Delfi Latvia - 'smart.delfi.lv' => array('Delfi lv', 'q', 'find?q={k}'), - - // Delfi - 'otsing.delfi.ee' => array('Delfi EE', 'q', 'find?q={k}'), - - // Digg - 'digg.com' => array('Digg', 's', 'search?s={k}'), - - // dir.com - 'fr.dir.com' => array('dir.com', 'req'), - - // dmoz - 'dmoz.org' => array('dmoz', 'search'), - 'editors.dmoz.org' => array('dmoz'), - - // DuckDuckGo - 'duckduckgo.com' => array('DuckDuckGo', 'q', '?q={k}'), - 'r.duckduckgo.com' => array('DuckDuckGo'), - - // earthlink - 'search.earthlink.net' => array('Earthlink', 'q', 'search?q={k}'), - - // Ecosia (powered by Bing) - 'ecosia.org' => array('Ecosia', 'q', 'search.php?q={k}'), - - // Eniro - 'www.eniro.se' => array('Eniro', array('q', 'search_word'), 'query?q={k}'), - - // Eurip - 'www.eurip.com' => array('Eurip', 'q', 'search/?q={k}'), - - // Euroseek - 'www.euroseek.com' => array('Euroseek', 'string', 'system/search.cgi?string={k}'), - - // Everyclick - 'www.everyclick.com' => array('Everyclick', 'keyword'), - - // Excite - 'search.excite.it' => array('Excite', 'q', 'web/?q={k}'), - 'search.excite.fr' => array('Excite'), - 'search.excite.de' => array('Excite'), - 'search.excite.co.uk' => array('Excite'), - 'search.excite.es' => array('Excite'), - 'search.excite.nl' => array('Excite'), - 'msxml.excite.com' => array('Excite', '/\/[^\/]+\/ws\/results\/[^\/]+\/([^\/]+)/'), - 'www.excite.co.jp' => array('Excite', 'search', 'search.gw?search={k}', 'SHIFT_JIS'), - - // Exalead - 'www.exalead.fr' => array('Exalead', 'q', 'search/results?q={k}'), - 'www.exalead.com' => array('Exalead'), - - // eo - 'eo.st' => array('eo', 'x_query', 'cgi-bin/eolost.cgi?x_query={k}'), - - // Facebook - 'www.facebook.com' => array('Facebook', 'q', 'search/?q={k}'), - - // Fast Browser Search - 'www.fastbrowsersearch.com' => array('Fast Browser Search', 'q', 'results/results.aspx?q={k}'), - - // Francite - 'recherche.francite.com' => array('Francite', 'name'), - - // Findhurtig - 'www.findhurtig.dk' => array('Findhurtig', 'q', 'web?q={k}'), - - // Fireball - 'www.fireball.de' => array('Fireball', 'q', 'ajax.asp?q={k}'), - - // Firstfind - 'www.firstsfind.com' => array('Firstsfind', 'qry'), - - // Fixsuche - 'www.fixsuche.de' => array('Fixsuche', 'q'), - - // Flix - 'www.flix.de' => array('Flix.de', 'keyword'), - - // Fooooo - 'search.fooooo.com' => array('Fooooo', 'q', 'web/?q={k}'), - - // Forestle - 'forestle.org' => array('Forestle', 'q', 'search.php?q={k}'), - '{}.forestle.org' => array('Forestle'), - 'forestle.mobi' => array('Forestle'), - - // Free - 'search.free.fr' => array('Free', array('q', 'qs')), - 'search1-2.free.fr' => array('Free'), - 'search1-1.free.fr' => array('Free'), - - // Freecause - 'search.freecause.com' => array('FreeCause', 'p', '?p={k}'), - - // Freenet - 'suche.freenet.de' => array('Freenet', array('query', 'Keywords'), 'suche/?query={k}'), - - // FriendFeed - 'friendfeed.com' => array('FriendFeed', 'q', 'search?q={k}'), - - // GAIS - 'gais.cs.ccu.edu.tw' => array('GAIS', 'q', 'search.php?q={k}'), - - // Genieo - 'search.genieo.com' => array('Genieo', 'q', '&q={k}'), - - // Geona - 'geona.net' => array('Geona', 'q', 'search?q={k}'), - - // Gigablast - 'www.gigablast.com' => array('Gigablast', 'q', 'search?q={k}'), - 'dir.gigablast.com' => array('Gigablast (Directory)', 'q'), - - // Gnadenmeer - 'www.gnadenmeer.de' => array('Gnadenmeer', 'keyword'), - - // Gomeo - 'www.gomeo.com' => array('Gomeo', array('Keywords', '/\/search\/([^\/]+)/'), '/search/{k}'), - - // goo - 'search.goo.ne.jp' => array('goo', 'MT', 'web.jsp?MT={k}'), - 'ocnsearch.goo.ne.jp' => array('goo'), - - // Google - 'google.com' => array('Google', 'q', 'search?q={k}'), - 'google.{}' => array('Google'), - 'www2.google.com' => array('Google'), - 'ipv6.google.com' => array('Google'), - 'go.google.com' => array('Google'), - - // Google vs typo squatters - 'wwwgoogle.com' => array('Google'), - 'wwwgoogle.{}' => array('Google'), - 'gogole.com' => array('Google'), - 'gogole.{}' => array('Google'), - 'gppgle.com' => array('Google'), - 'gppgle.{}' => array('Google'), - 'googel.com' => array('Google'), - 'googel.{}' => array('Google'), - - // Powered by Google - 'search.avg.com' => array('Google'), - 'isearch.avg.com' => array('Google'), - 'search.chedot.com' => array('Google', 'text'), - 'www.cnn.com' => array('Google', 'query'), - 'darkoogle.com' => array('Google'), - 'search.darkoogle.com' => array('Google'), - 'search.foxtab.com' => array('Google'), - 'www.gooofullsearch.com' => array('Google', 'Keywords'), - 'search.hiyo.com' => array('Google'), - 'search.incredimail.com' => array('Google'), - 'search1.incredimail.com' => array('Google'), - 'search2.incredimail.com' => array('Google'), - 'search3.incredimail.com' => array('Google'), - 'search4.incredimail.com' => array('Google'), - 'search.sweetim.com' => array('Google'), - 'www.fastweb.it' => array('Google'), - 'search.juno.com' => array('Google', 'query'), - 'find.tdc.dk' => array('Google'), - 'it.luna.tv' => array('Google'), - 'searchresults.verizon.com' => array('Google'), - 'search.walla.co.il' => array('Google'), - 'search.alot.com' => array('Google'), - 'suche.gmx.net' => array('Google', 'q', 'web?q={k}'), - 'search.incredibar.com' => array('Google', 'q', 'search.php?q={k}'), - 'www.delta-search.com' => array('Google', 'q'), - 'www1.delta-search.com' => array('Google', 'q'), - 'search.1und1.de' => array('Google', 'q', 'web?q={k}'), - 'suche.1und1.de' => array('Google', 'q', 'web?q={k}'), - 'search.zonealarm.com' => array('Google'), - 'start.lenovo.com' => array('Google', 'q', 'search/index.php?q={k}'), - 'wow.com' => array('Google'), - '{}.wow.com' => array('Google'), - 'search.leonardo.it' => array('Google'), - 'www.optuszoo.com.au' => array('Google'), - 'search.dolphin-browser.jp' => array('Google'), - 'netlavis.azione.jp' => array('Google'), - 'search.nan.so' => array('Google'), - 'cgi2.nintendo.co.jp' => array('Google', 'gsc.q'), - 'search.smt.docomo.ne.jp' => array('Google', 'MT'), - 'image.search.smt.docomo.ne.jp' => array('Google', 'MT'), - 'gfsoso.com' => array('Google', 'q'), - 'searches.safehomepage.com' => array('Google', 'q'), - 'searches.f-secure.com' => array('Google', 'query', 'search?query={k}'), - - // Google Cache - 'webcache.googleusercontent.com' => array('Google', '/\/search\?q=cache:[A-Za-z0-9]+:[^+]+([^&]+)/', 'search?q={k}'), - - // Google SSL - 'encrypted.google.com' => array('Google SSL', 'q', 'search?q={k}'), - - // Google Blogsearch - 'blogsearch.google.com' => array('Google Blogsearch', 'q', 'blogsearch?q={k}'), - 'blogsearch.google.{}' => array('Google Blogsearch'), - - // Google Custom Search - 'google.com/cse' => array('Google Custom Search', array('q', 'query')), - 'google.{}/cse' => array('Google Custom Search'), - 'google.com/custom' => array('Google Custom Search'), - 'google.{}/custom' => array('Google Custom Search'), - - // Google Translation - 'translate.google.com' => array('Google Translations', 'q'), - - // Google Images - 'images.google.com' => array('Google Images', 'q', 'images?q={k}'), - 'images.google.{}' => array('Google Images'), - - // Google Maps - 'maps.google.com' => array('Google Maps', 'q', 'maps?q={k}'), - 'maps.google.{}' => array('Google Maps'), - - // Google News - 'news.google.com' => array('Google News', 'q'), - 'news.google.{}' => array('Google News'), - - // Google Shopping - 'google.com/products' => array('Google Shopping', 'q', '?q={k}&tbm=shop'), - 'google.{}/products' => array('Google Shopping'), - - // Google syndicated search - 'googlesyndicatedsearch.com' => array('Google syndicated search', 'q'), - - // Google Video - 'video.google.com' => array('Google Video', 'q', 'search?q={k}&tbm=vid'), - - // Google Scholar - 'scholar.google.com' => array('Google Scholar', 'q', 'scholar?q={k}'), - 'scholar.google.{}' => array('Google Scholar'), - - // Google Wireless Transcoder - // - does not appear to execute JavaScript -// 'google.com/gwt/n' => array('Google Wireless Transcoder'), - - // Goyellow.de - 'www.goyellow.de' => array('GoYellow.de', 'MDN'), - - // Gule Sider - 'www.gulesider.no' => array('Gule Sider', 'q'), - - // Haosou - 'www.haosou.com' => array('Haosou', 'q', 's?q={k}'), - - // HighBeam - 'www.highbeam.com' => array('HighBeam', 'q', 'Search.aspx?q={k}'), - - // Hit-Parade - 'req.hit-parade.com' => array('Hit-Parade', 'p7', 'general/recherche.asp?p7={k}'), - 'class.hit-parade.com' => array('Hit-Parade'), - 'www.hit-parade.com' => array('Hit-Parade'), - - // Holmes.ge - 'holmes.ge' => array('Holmes', 'q', 'search.htm?q={k}'), - - // Hooseek.com - 'www.hooseek.com' => array('Hooseek', 'recherche', 'web?recherche={k}'), - - // Hotbot - 'www.hotbot.com' => array('Hotbot', 'query'), - - // Icerocket - 'blogs.icerocket.com' => array('Icerocket', 'q', 'search?q={k}'), - - // ICQ - 'www.icq.com' => array('ICQ', 'q', 'search/results.php?q={k}'), - 'search.icq.com' => array('ICQ'), - - // Ilse - 'www.ilse.nl' => array('Ilse NL', 'search_for', '?search_for={k}'), - - // iMesh - 'search.imesh.com' => array('iMesh', array('q', 'si'), 'web?q={k}'), - - // Inbox.com - 'www2.inbox.com' => array('Inbox', 'q', 'search/results1.aspx?q={k}'), - - // InfoSpace (and related web properties) - 'infospace.com' => array('InfoSpace', 'q', '/search/web?q={k}'), - 'dogpile.com' => array('InfoSpace'), - 'tattoodle.com' => array('InfoSpace'), - 'metacrawler.com' => array('InfoSpace'), - 'webfetch.com' => array('InfoSpace'), - 'webcrawler.com' => array('InfoSpace'), - 'search.kiwee.com' => array('InfoSpace'), - 'searches.vi-view.com' => array('InfoSpace'), - 'search.webssearches.com' => array('InfoSpace'), - 'search.fbdownloader.com' => array('InfoSpace'), - 'searches3.globososo.com' => array('InfoSpace'), - - // old infospace system - 'wsdsold.infospace.com' => array('InfoSpace', '/\/[^\/]+\/ws\/results\/[^\/]+\/([^\/]+)/', 'pemonitorhosted/ws/results/Web/{k}/1/417/TopNavigation/Source/'), - - // Powered by InfoSpace - 'isearch.babylon.com' => array('InfoSpace', 'q'), - 'start.facemoods.com' => array('InfoSpace', 's'), - 'start.funmoods.com' => array('InfoSpace', 'q'), - 'search.magentic.com' => array('InfoSpace', 'q'), - 'search.searchcompletion.com' => array('InfoSpace', 'q'), - 'www.searchmobileonline.com' => array('InfoSpace', 'q'), - 'isearch.glarysoft.com' => array('InfoSpace', 'q'), - 'search.chatzum.com' => array('InfoSpace', 'q'), - 'home.speedbit.com' => array('InfoSpace', 'q'), - 'search.b1.org' => array('InfoSpace', 'q'), - 'searchya.com' => array('InfoSpace', 'q'), - 'search.handycafe.com' => array('InfoSpace', 'q'), - 'search.v9.com' => array('InfoSpace', 'q'), - 'search.iminent.com' => array('InfoSpace', 'q'), - 'utorrent.inspsearch.com' => array('InfoSpace', 'q'), - - /* - * Other InfoSpace powered metasearches are handled in Common::extractSearchEngineInformationFromUrl() - * - * This includes sites such as: - * - search.nation.com - * - ws.copernic.com - * - result.iminent.com - */ - - // Interia - 'www.google.interia.pl' => array('Interia', 'q', 'szukaj?q={k}'), - - // I-play - 'start.iplay.com' => array('I-play', 'q', 'searchresults.aspx?q={k}'), - - // Ixquick - 'ixquick.com' => array('Ixquick', 'query'), - 'www.eu.ixquick.com' => array('Ixquick'), - 'ixquick.de' => array('Ixquick'), - 'www.ixquick.de' => array('Ixquick'), - 'us.ixquick.com' => array('Ixquick'), - 's1.us.ixquick.com' => array('Ixquick'), - 's2.us.ixquick.com' => array('Ixquick'), - 's3.us.ixquick.com' => array('Ixquick'), - 's4.us.ixquick.com' => array('Ixquick'), - 's5.us.ixquick.com' => array('Ixquick'), - 'eu.ixquick.com' => array('Ixquick'), - 's8-eu.ixquick.com' => array('Ixquick'), - 's1-eu.ixquick.de' => array('Ixquick'), - 's2-eu4.ixquick.com' => array('Ixquick'), - 's5-eu4.ixquick.com' => array('Ixquick'), - - // Jyxo - 'jyxo.1188.cz' => array('Jyxo', 'q', 's?q={k}'), - - // Jungle Spider - 'www.jungle-spider.de' => array('Jungle Spider', 'q'), - - // Jungle key - 'junglekey.com' => array('Jungle Key', 'query', 'search.php?query={k}&type=web&lang=en'), - 'junglekey.fr' => array('Jungle Key'), - - // K9 Safe Search - 'k9safesearch.com' => array('K9 Safe Search', 'q', 'search.jsp?q={k}'), - - // Kataweb - 'www.kataweb.it' => array('Kataweb', 'q'), - - // Kensaq - 'www.kensaq.com' => array('Kensaq', 'q', 'web?q={k}'), - - // Kvasir - 'www.kvasir.no' => array('Kvasir', 'q', 'alle?q={k}'), - - // 묻지마 검색 - 'kwzf.net' => array('묻지마 검색', 'search', '#search={k}'), - - // Latne - 'www.latne.lv' => array('Latne', 'q', 'siets.php?q={k}'), - - // La Toile Du Québec via Google - 'www.toile.com' => array('La Toile Du Québec (Google)', 'q', 'search?q={k}'), - 'web.toile.com' => array('La Toile Du Québec (Google)'), - - // LookAny - 'www.lookany.com' => array('LookAny', '/(?:search|images|videos)\/([^\/]+)/'), - - // Looksmart - 'www.looksmart.com' => array('Looksmart', 'key'), - - // Lo.st (Enhanced by Google) - 'lo.st' => array('Lo.st', 'x_query', 'cgi-bin/eolost.cgi?x_query={k}'), - - // Lycos - 'search.lycos.com' => array('Lycos', 'query', '?query={k}'), - 'lycos.{}' => array('Lycos'), - - // maailm.com - 'www.maailm.com' => array('maailm.com', 'tekst'), - - // Mail.ru - 'go.mail.ru' => array('Mailru', 'q', 'search?rch=e&q={k}', array('UTF-8', 'windows-1251')), - - // Mamma - 'www.mamma.com' => array('Mamma', 'query', 'result.php?q={k}'), - 'mamma75.mamma.com' => array('Mamma'), - - // Meta - 'meta.ua' => array('Meta.ua', 'q', 'search.asp?q={k}'), - - // MetaCrawler.de - 's1.metacrawler.de' => array('MetaCrawler DE', 'qry', '?qry={k}'), - 's2.metacrawler.de' => array('MetaCrawler DE'), - 's3.metacrawler.de' => array('MetaCrawler DE'), - - // Metager - 'meta.rrzn.uni-hannover.de' => array('Metager', 'eingabe', 'meta/cgi-bin/meta.ger1?eingabe={k}'), - 'www.metager.de' => array('Metager'), - 'metager.de' => array('Metager'), - - // Metager2 - 'metager2.de' => array('Metager2', 'q', 'search/index.php?q={k}'), - - // Meinestadt - 'www.meinestadt.de' => array('Meinestadt.de', 'words'), - - // Mister Wong - 'www.mister-wong.com' => array('Mister Wong', 'keywords', 'search/?keywords={k}'), - 'www.mister-wong.de' => array('Mister Wong'), - - // Monstercrawler - 'www.monstercrawler.com' => array('Monstercrawler', 'qry'), - - // Mozbot - 'www.mozbot.fr' => array('mozbot', 'q', 'results.php?q={k}'), - 'www.mozbot.co.uk' => array('mozbot'), - 'www.mozbot.com' => array('mozbot'), - - // El Mundo - 'ariadna.elmundo.es' => array('El Mundo', 'q'), - - // MySpace - 'searchservice.myspace.com' => array('MySpace', 'qry', 'index.cfm?fuseaction=sitesearch.results&type=Web&qry={k}'), - - // MySearch / MyWay / MyWebSearch (default: powered by Ask.com) - 'www.mysearch.com' => array('MyWebSearch', array('searchfor', 'searchFor'), 'search/Ajmain.jhtml?searchfor={k}'), - 'ms114.mysearch.com' => array('MyWebSearch'), - 'ms146.mysearch.com' => array('MyWebSearch'), - 'kf.mysearch.myway.com' => array('MyWebSearch'), - 'ki.mysearch.myway.com' => array('MyWebSearch'), - 'search.myway.com' => array('MyWebSearch'), - 'search.mywebsearch.com' => array('MyWebSearch'), - - // Najdi - 'www.najdi.si' => array('Najdi.si', 'q', 'search.jsp?q={k}'), - - // Nate - 'search.nate.com' => array('Nate', 'q', 'search/all.html?q={k}', 'EUC-KR'), - - // Naver - 'search.naver.com' => array('Naver', 'query', 'search.naver?query={k}'), - - // Needtofind - 'ko.search.need2find.com' => array('Needtofind', 'searchfor', 'search/AJmain.jhtml?searchfor={k}'), - - // Neti - 'www.neti.ee' => array('Neti', 'query', 'cgi-bin/otsing?query={k}', 'iso-8859-1'), - - // Nifty - 'search.nifty.com' => array('Nifty', array('q', 'Text'), 'websearch/search?q={k}'), - 'search.azby.fmworld.net' => array('Nifty'), - 'videosearch.nifty.com' => array('Nifty Videos', 'kw', 'search?kw={k}'), - - // Nigma - 'nigma.ru' => array('Nigma', 's', 'index.php?s={k}'), - - // Onet - 'szukaj.onet.pl' => array('Onet.pl', 'qt', 'query.html?qt={k}'), - - // Online.no - 'online.no' => array('Online.no', 'q', 'google/index.jsp?q={k}'), - - // Opplysningen 1881 - 'www.1881.no' => array('Opplysningen 1881', 'Query', 'Multi/?Query={k}'), - - // Orange - 'busca.orange.es' => array('Orange', 'q', 'search?q={k}'), - 'lemoteur.ke.voila.fr' => array('Orange', 'kw', '?kw={k}'), - - // Paperball - 'www.paperball.de' => array('Paperball', 'q', 'suche/s/?q={k}'), - - // PeoplePC - 'search.peoplepc.com' => array('PeoplePC', 'q', 'search?q={k}'), - - // PeopleCheck - 'extern.peoplecheck.de' => array('PeopleCheck', 'q', 'link.php?q={k}'), - - // Picsearch - 'www.picsearch.com' => array('Picsearch', 'q', 'index.cgi?q={k}'), - - // Plazoo - 'www.plazoo.com' => array('Plazoo', 'q'), - - // PlusNetwork - 'plusnetwork.com' => array('PlusNetwork', 'q', '?q={k}'), - - // Poisk.Ru - 'poisk.ru' => array('Poisk.Ru', 'text', 'cgi-bin/poisk?text={k}', 'windows-1251'), - - // qip - 'search.qip.ru' => array('qip.ru', 'query', 'search?query={k}'), - - // Qualigo - 'www.qualigo.at' => array('Qualigo', 'q'), - 'www.qualigo.ch' => array('Qualigo'), - 'www.qualigo.de' => array('Qualigo'), - 'www.qualigo.nl' => array('Qualigo'), - - // Qwant - 'www.qwant.com' => array('Qwant', 'q'), - - // Rakuten - 'websearch.rakuten.co.jp' => array('Rakuten', 'qt', 'WebIS?qt={k}'), - - // Rambler - 'nova.rambler.ru' => array('Rambler', array('query', 'words'), 'search?query={k}'), - - // RPMFind - 'rpmfind.net' => array('rpmfind', 'query', 'linux/rpm2html/search.php?query={k}'), - 'fr2.rpmfind.net' => array('rpmfind'), - - // Road Runner Search - 'search.rr.com' => array('Road Runner', 'q', '?q={k}'), - - // Sapo - 'pesquisa.sapo.pt' => array('Sapo', 'q', '?q={k}'), - - // scour.com - 'scour.com' => array('Scour.com', '/search\/[^\/]+\/(.*)/', 'search/web/{k}'), - - // Search.com - 'www.search.com' => array('Search.com', 'q', 'search?q={k}'), - - // Search.ch - 'www.search.ch' => array('Search.ch', 'q', '?q={k}'), - - // Searchalot - 'searchalot.com' => array('Searchalot', 'q', '?q={k}'), - - // SearchCanvas - 'www.searchcanvas.com' => array('SearchCanvas', 'q', 'web?q={k}'), - - // Searchy - 'www.searchy.co.uk' => array('Searchy', 'q', 'index.html?q={k}'), - - // Setooz - // 2010-09-13: the mismatches are because subdomains are language codes - // (not country codes) - 'bg.setooz.com' => array('Setooz', 'query', 'search?query={k}'), - 'da.setooz.com' => array('Setooz'), - 'el.setooz.com' => array('Setooz'), - 'fa.setooz.com' => array('Setooz'), - 'ur.setooz.com' => array('Setooz'), - '{}.setooz.com' => array('Setooz'), - - // Seznam - 'search.seznam.cz' => array('Seznam', 'q', '?q={k}'), - - // Seznam Videa (Video) - 'videa.seznam.cz' => array('Seznam Videa', 'q', '?q={k}'), - - // Sharelook - 'www.sharelook.fr' => array('Sharelook', 'keyword'), - - // Skynet - 'www.skynet.be' => array('Skynet', 'q', 'services/recherche/google?q={k}'), - - // sm.cn - 'm.sm.cn' => array('sm.cn', 'q', 's?q={k}'), - 'm.sp.sm.cn' => array('sm.cn'), - - // sm.de - 'www.sm.de' => array('sm.de', 'q', '?q={k}'), - - // SmartAdressbar - 'search.smartaddressbar.com' => array('SmartAddressbar', 's', '?s={k}'), - - // Snap.do - 'search.snap.do' => array('Snap.do', 'q', '?q={k}'), - - // SeeSaa - 'search.seesaa.jp' => array('SeeSaa', '/\/([^\/]+)\/index\.html/', '{k}/index.html'), - - // So-net - 'www.so-net.ne.jp' => array('So-net', 'query', 'search/web/?query={k}'), - 'video.so-net.ne.jp' => array('So-net Videos', 'kw', 'search/?kw={k}'), - - // Sogou - 'www.sogou.com' => array('Sogou', 'query', 'web?query={k}', 'gb2312'), - 'm.sogou.com' => array('Sogou', 'keyword'), - - // Softonic - 'search.softonic.com' => array('Softonic', 'q', 'default/default?q={k}'), - - // soso.com - 'www.soso.com' => array('Soso', 'w', 'q?w={k}', 'gb2312'), - - // sputnik.ru - 'www.sputnik.ru' => array('Sputnik', 'q', 'search?q={k}'), - - // Startpagina - 'startgoogle.startpagina.nl' => array('Startpagina (Google)', 'q', '?q={k}'), - - // Startsiden - 'www.startsiden.no' => array('Startsiden', 'q', 'sok/index.html?q={k}'), - - // suche.info - 'suche.info' => array('Suche.info', 'Keywords', 'suche.php?Keywords={k}'), - - // Suchmaschine.com - 'www.suchmaschine.com' => array('Suchmaschine.com', 'suchstr', 'cgi-bin/wo.cgi?suchstr={k}'), - - // Suchnase - 'www.suchnase.de' => array('Suchnase', 'q'), - - // Surf Canyon - 'surfcanyon.com' => array('Surf Canyon', 'q'), - - // talimba - 'www.talimba.com' => array('talimba', 'search', 'index.php?page=search/web&search={k}'), - - // TalkTalk - 'www.talktalk.co.uk' => array('TalkTalk', 'query', 'search/results.html?query={k}'), - - // Technorati - 'technorati.com' => array('Technorati', 'q', 'search?return=sites&authority=all&q={k}'), - - // Teoma - 'www.teoma.com' => array('Teoma', 'q', 'web?q={k}'), - - // Terra -- referrer does not contain search phrase (keywords) - 'buscador.terra.es' => array('Terra', 'query', 'Default.aspx?source=Search&query={k}'), - 'buscador.terra.cl' => array('Terra'), - 'buscador.terra.com.br' => array('Terra'), - - // Tiscali - 'search.tiscali.it' => array('Tiscali', array('q', 'key'), '?q={k}'), - 'search-dyn.tiscali.it' => array('Tiscali'), - 'hledani.tiscali.cz' => array('Tiscali', 'query'), - - // Tixuma - 'www.tixuma.de' => array('Tixuma', 'sc', 'index.php?mp=search&stp=&sc={k}&tg=0'), - - // T-Online - 'suche.t-online.de' => array('T-Online', 'q', 'fast-cgi/tsc?mandant=toi&context=internet-tab&q={k}'), - 'brisbane.t-online.de' => array('T-Online'), - 'navigationshilfe.t-online.de' => array('T-Online', 'q', 'dtag/dns/results?mode=search_top&q={k}'), - - // Toolbarhome - 'www.toolbarhome.com' => array('Toolbarhome', 'q', 'search.aspx?q={k}'), - 'vshare.toolbarhome.com' => array('Toolbarhome'), - - // Toppreise.ch - 'www.toppreise.ch' => array('Toppreise.ch', 'search', 'index.php?search={k}', 'ISO-8859-1'), - 'toppreise.ch' => array('Toppreise.ch', null, null, 'ISO-8859-1'), - 'fr.toppreise.ch' => array('Toppreise.ch', null, null, 'ISO-8859-1'), - 'de.toppreise.ch' => array('Toppreise.ch', null, null, 'ISO-8859-1'), - 'en.toppreise.ch' => array('Toppreise.ch', null, null, 'ISO-8859-1'), - - // Trouvez.com - 'www.trouvez.com' => array('Trouvez.com', 'query'), - - // TrovaRapido - 'www.trovarapido.com' => array('TrovaRapido', 'q', 'result.php?q={k}'), - - // Trusted-Search - 'www.trusted-search.com' => array('Trusted Search', 'w', 'search?w={k}'), - - // Twingly - 'www.twingly.com' => array('Twingly', 'q', 'search?q={k}'), - - // uol.com.br - 'busca.uol.com.br' => array('uol.com.br', 'q', '/web/?q={k}'), - - // URL.ORGanzier - 'www.url.org' => array('URL.ORGanzier', 'q', '?l=de&q={k}'), - - // Vinden - 'www.vinden.nl' => array('Vinden', 'q', '?q={k}'), - - // Vindex - 'www.vindex.nl' => array('Vindex', 'search_for', '/web?search_for={k}'), - 'search.vindex.nl' => array('Vindex'), - - // Virgilio - 'ricerca.virgilio.it' => array('Virgilio', 'qs', 'ricerca?qs={k}'), - 'ricercaimmagini.virgilio.it' => array('Virgilio'), - 'ricercavideo.virgilio.it' => array('Virgilio'), - 'ricercanews.virgilio.it' => array('Virgilio'), - 'mobile.virgilio.it' => array('Virgilio', 'qrs'), - - // Voila - 'search.ke.voila.fr' => array('Voila', 'rdata', 'S/voila?rdata={k}'), - 'www.lemoteur.fr' => array('Voila'), // uses voila search - - // Volny - 'web.volny.cz' => array('Volny', 'search', 'fulltext/?search={k}', 'windows-1250'), - - // Walhello - 'www.walhello.info' => array('Walhello', 'key', 'search?key={k}'), - 'www.walhello.com' => array('Walhello'), - 'www.walhello.de' => array('Walhello'), - 'www.walhello.nl' => array('Walhello'), - - // Web.de - 'suche.web.de' => array('Web.de', array('su', 'q'), 'search/web/?su={k}'), - 'm.suche.web.de' => array('Web.de'), - - // Web.nl - 'www.web.nl' => array('Web.nl', 'zoekwoord'), - - // Weborama - 'www.weborama.fr' => array('weborama', 'QUERY'), - - // WebSearch - 'www.websearch.com' => array('WebSearch', array('qkw', 'q'), 'search/results2.aspx?q={k}'), - - // Wedoo - // 2011-02-15 - keyword no longer appears to be in Referrer URL; candidate for removal? - 'fr.wedoo.com' => array('Wedoo', 'keyword'), - 'en.wedoo.com' => array('Wedoo'), - 'es.wedoo.com' => array('Wedoo'), - - // Winamp (Enhanced by Google) - 'search.winamp.com' => array('Winamp', 'q', 'search/search?q={k}'), - - // Witch - 'www.witch.de' => array('Witch', 'search', 'search-result.php?cn=0&search={k}'), - - // Wirtualna Polska - 'szukaj.wp.pl' => array('Wirtualna Polska', 'szukaj', 'http://szukaj.wp.pl/szukaj.html?szukaj={k}'), - - // Woopie - 'www.woopie.jp' => array('Woopie', 'kw', 'search?kw={k}'), - - // WWW - 'search.www.ee' => array('www värav', 'query'), - - // X-recherche - 'www.x-recherche.com' => array('X-Recherche', 'MOTS', 'cgi-bin/websearch?MOTS={k}'), - - // Yahoo! Japan - 'search.yahoo.co.jp' => array('Yahoo! Japan', array('p', 'vp'), 'search?p={k}'), - 'jp.hao123.com' => array('Yahoo! Japan', 'query'), - 'home.kingsoft.jp' => array('Yahoo! Japan', 'keyword'), - 'video.search.yahoo.co.jp' => array('Yahoo! Japan Videos', 'p', 'search?p={k}'), - 'image.search.yahoo.co.jp' => array('Yahoo! Japan Images', 'p', 'search?p={k}'), - - // Yahoo - 'search.yahoo.com' => array('Yahoo!', array('p', 'q'), 'search?p={k}'), -// '*.search.yahoo.com' => array('Yahoo!'), // see built-in helper in Common.php - 'yahoo.com' => array('Yahoo!'), - 'yahoo.{}' => array('Yahoo!'), - '{}.yahoo.com' => array('Yahoo!'), - 'cade.yahoo.com' => array('Yahoo!'), - 'espanol.yahoo.com' => array('Yahoo!'), - 'qc.yahoo.com' => array('Yahoo!'), - 'one.cn.yahoo.com' => array('Yahoo!'), - - // Powered by Yahoo APIs - 'www.cercato.it' => array('Yahoo!', 'q'), - 'search.offerbox.com' => array('Yahoo!', 'q'), - 'www.benefind.de' => array('Yahoo!', 'q'), - - // Powered by Yahoo! Search Marketing (Overture) - 'ys.mirostart.com' => array('Yahoo!', 'q'), - - // Yahoo! Directory - 'search.yahoo.com/search/dir' => array('Yahoo! Directory', 'p', '?p={k}'), -// '{}.dir.yahoo.com' => array('Yahoo! Directory'), - - // Yahoo! Images - 'images.search.yahoo.com' => array('Yahoo! Images', array('p', 'va'), 'search/images?p={k}'), -// '*.images.search.yahoo.com'=> array('Yahoo! Images'), // see built-in helper in Common.php - '{}.images.yahoo.com' => array('Yahoo! Images'), - 'cade.images.yahoo.com' => array('Yahoo! Images'), - 'espanol.images.yahoo.com' => array('Yahoo! Images'), - 'qc.images.yahoo.com' => array('Yahoo! Images'), - - // Yam - 'search.yam.com' => array('Yam', 'k', 'Search/Web/?SearchType=web&k={k}'), - - // Yandex - 'yandex.ru' => array('Yandex', 'text', 'yandsearch?text={k}'), - 'yandex.com' => array('Yandex'), - 'yandex.{}' => array('Yandex'), - - // Yandex Images - 'images.yandex.ru' => array('Yandex Images', 'text', 'yandsearch?text={k}'), - 'images.yandex.com' => array('Yandex Images'), - 'images.yandex.{}' => array('Yandex Images'), - - // Yasni - 'www.yasni.de' => array('Yasni', 'query'), - 'www.yasni.com' => array('Yasni'), - 'www.yasni.co.uk' => array('Yasni'), - 'www.yasni.ch' => array('Yasni'), - 'www.yasni.at' => array('Yasni'), - - // Yatedo - 'www.yatedo.com' => array('Yatedo', 'q', 'search/profil?q={k}'), - 'www.yatedo.fr' => array('Yatedo'), - - // Yellowmap - 'yellowmap.de' => array('Yellowmap', ' '), - - // Yippy - 'search.yippy.com' => array('Yippy', 'query', 'search?query={k}'), - - // YouGoo - 'www.yougoo.fr' => array('YouGoo', 'q', '?cx=search&q={k}'), - - // Zapmeta - 'www.zapmeta.com' => array('Zapmeta', array('q', 'query'), '?q={k}'), - 'zapmeta.{}' => array('Zapmeta'), - 'uk.zapmeta.com' => array('Zapmeta'), - 'ar.zapmeta.com' => array('Zapmeta'), - 'au.zapmeta.com' => array('Zapmeta'), - 'ca.zapmeta.com' => array('Zapmeta'), - 'fi.zapmeta.com' => array('Zapmeta'), - 'no.zapmeta.com' => array('Zapmeta'), - 'tr.zapmeta.com' => array('Zapmeta'), - - // Zoek - 'www3.zoek.nl' => array('Zoek', 'q'), - - // Zhongsou - 'p.zhongsou.com' => array('Zhongsou', 'w', 'p?w={k}'), - - // Zoeken - 'www.zoeken.nl' => array('Zoeken', 'q', '?q={k}'), - - // Zoohoo - 'zoohoo.cz' => array('Zoohoo', 'q', '?q={k}', 'windows-1250'), - - // Zoznam - 'www.zoznam.sk' => array('Zoznam', 's', 'hladaj.fcgi?s={k}&co=svet'), - - // Zxuso - 'www.zxuso.com' => array('Zxuso', 'wd', 'ri/?wd={k}'), - ); -} diff --git a/core/DataFiles/Socials.php b/core/DataFiles/Socials.php deleted file mode 100755 index 92b131b496..0000000000 --- a/core/DataFiles/Socials.php +++ /dev/null @@ -1,230 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - * - */ - -if (!isset($GLOBALS['Piwik_socialUrl'])) { - // Note: the key of the array should have max 3 elements eg. sub.domain.ext - $GLOBALS['Piwik_socialUrl'] = array( - - // Facebook - 'facebook.com' => 'Facebook', - 'fb.me' => 'Facebook', - - // Ozone - 'qzone.qq.com' => 'Qzone', - - // Haboo - 'habbo.com' => 'Haboo', - - // Twitter - 'twitter.com' => 'Twitter', - 't.co' => 'Twitter', - - // Renren - 'renren.com' => 'Renren', - - // Windows Live Spaces - 'login.live.com' => 'Windows Live Spaces', - - // LinkedIn - 'linkedin.com' => 'LinkedIn', - - // Bebo - 'bebo.com' => 'Bebo', - - // Vkontakte - 'vk.com' => 'Vkontakte', - 'vkontakte.ru' => 'Vkontakte', - - // Tagged - 'login.tagged.com' => 'Tagged', - - // Orkut - 'orkut.com' => 'Orkut', - - // Myspace - 'myspace.com' => 'Myspace', - - // Frinedster - 'friendster.com' => 'Friendster', - - // Badoo - 'badoo.com' => 'Badoo', - - // hi5 - 'hi5.com' => 'hi5', - - // Netlog - 'netlog.com' => 'Netlog', - - // Flixster - 'flixster.com' => 'Flixster', - - // MyLife - 'mylife.ru' => 'MyLife', - - // Classmates.com - 'classmates.com' => 'Classmates.com', - - // Github - 'github.com' => 'Github', - - // Google+ - 'plus.google.com' => 'Google%2B', - 'url.google.com' => 'Google%2B', - - // douban - 'douban.com' => 'douban', - - // dribbble - 'dribbble.com' => 'dribbble', - - // Odnoklassniki - 'odnoklassniki.ru' => 'Odnoklassniki', - - // Viadeo - 'viadeo.com' => 'Viadeo', - - // Flickr - 'flickr.com' => 'Flickr', - - // WeeWorld - 'weeworld.com' => 'WeeWorld', - - // Last.fm - 'last.fm' => 'Last.fm', - 'lastfm.ru' => 'Last.fm', - 'lastfm.de' => 'Last.fm', - 'lastfm.es' => 'Last.fm', - 'lastfm.fr' => 'Last.fm', - 'lastfm.it' => 'Last.fm', - 'lastfm.jp' => 'Last.fm', - 'lastfm.pl' => 'Last.fm', - 'lastfm.com.br' => 'Last.fm', - 'lastfm.se' => 'Last.fm', - 'lastfm.com.tr' => 'Last.fm', - - // MyHeritage - 'myheritage.com' => 'MyHeritage', - - // Xanga - 'xanga.com' => 'Xanga', - - // Mixi - 'mixi.jp' => 'Mixi', - - // Cyworld - 'global.cyworld.com' => 'Cyworld', - - // Gaia Online - 'gaiaonline.com' => 'Gaia Online', - - // Skyrock - 'skyrock.com' => 'Skyrock', - - // BlackPlanet - 'blackplanet.com' => 'BlackPlanet', - - // myYearbook - 'myyearbook.com' => 'myYearbook', - - // Fotolog - 'fotolog.com' => 'Fotolog', - - // Friends Reunited - 'friendsreunited.com' => 'Friends Reunited', - - // LiveJournal - 'livejournal.ru' => 'LiveJournal', - 'livejournal.com' => 'LiveJournal', - - // StudiVZ/MeinVZ - 'studivz.net' => 'StudiVZ', - 'meinvz.net' => 'MeinVZ', - - // StackOverflow - 'stackoverflow.com' => 'StackOverflow', - - // Sonico.com - 'sonico.com' => 'Sonico.com', - - // Pinterest - 'pinterest.com' => 'Pinterest', - - // Plaxo - 'plaxo.com' => 'Plaxo', - - // Geni.com - 'geni.com' => 'Geni.com', - - // Tuenti - 'tuenti.com' => 'Tuenti', - - // XING - 'xing.com' => 'XING', - - // Taringa! - 'taringa.net' => 'Taringa!', - - // Nasza-klasa.pl - 'nk.pl' => 'Nasza-klasa.pl', - - // StumbleUpon - 'stumbleupon.com' => 'StumbleUpon', - - // Sourceforge - 'sourceforge.net' => 'SourceForge', - - // Hyves - 'hyves.nl' => 'Hyves', - - // WAYN - 'wayn.com' => 'WAYN', - - // Buzznet - 'buzznet.com' => 'Buzznet', - - // Multiply - 'multiply.com' => 'Multiply', - - // Foursquare - 'foursquare.com' => 'Foursquare', - - // vkrugudruzei.ru - 'vkrugudruzei.ru' => 'vkrugudruzei.ru', - - // my.mail.ru - 'my.mail.ru' => 'my.mail.ru', - - //MoiKrug.ru - 'moikrug.ru' => 'moikrug.ru', - - // Reddit - 'reddit.com' => 'reddit', - - // HackerNews - 'news.ycombinator.com' => 'Hacker News', - - // Identi.ca - 'identi.ca' => 'identi.ca', - - // Weibo - 'weibo.com' => 'Weibo', - 't.cn' => 'Weibo', - - // YouTube - 'youtube.com' => 'YouTube', - 'youtu.be' => 'YouTube', - - // Vimeo - 'vimeo.com' => 'Vimeo', - - //tumblr - 'tumblr.com' => 'tumblr', - ); -} diff --git a/core/Metrics/Formatter.php b/core/Metrics/Formatter.php index 077646ba00..d62db65f31 100644 --- a/core/Metrics/Formatter.php +++ b/core/Metrics/Formatter.php @@ -72,10 +72,15 @@ class Formatter // Display 01:45:17 time format if ($displayTimeAsSentence === false) { - $hours = floor($numberOfSeconds / 3600); - $minutes = floor(($reminder = ($numberOfSeconds - $hours * 3600)) / 60); + $days = floor($numberOfSeconds / 86400); + $hours = floor(($reminder = ($numberOfSeconds - $days * 86400)) / 3600); + $minutes = floor(($reminder = ($reminder - $hours * 3600)) / 60); $seconds = floor($reminder - $minutes * 60); - $time = sprintf("%02s", $hours) . ':' . sprintf("%02s", $minutes) . ':' . sprintf("%02s", $seconds); + if ($days == 0) { + $time = sprintf("%02s", $hours) . ':' . sprintf("%02s", $minutes) . ':' . sprintf("%02s", $seconds); + } else { + $time = sprintf(Piwik::translate('Intl_NDays'), $days) . " " . sprintf("%02s", $hours) . ':' . sprintf("%02s", $minutes) . ':' . sprintf("%02s", $seconds); + } $centiSeconds = ($numberOfSeconds * 100) % 100; if ($centiSeconds) { $time .= '.' . sprintf("%02s", $centiSeconds); diff --git a/core/UrlHelper.php b/core/UrlHelper.php index 4a0ac0fa0a..66a0e64e25 100644 --- a/core/UrlHelper.php +++ b/core/UrlHelper.php @@ -259,242 +259,6 @@ class UrlHelper } /** - * Extracts a keyword from a raw not encoded URL. - * Will only extract keyword if a known search engine has been detected. - * Returns the keyword: - * - in UTF8: automatically converted from other charsets when applicable - * - strtolowered: "QUErY test!" will return "query test!" - * - trimmed: extra spaces before and after are removed - * - * Lists of supported search engines can be found in /core/DataFiles/SearchEngines.php - * The function returns false when a keyword couldn't be found. - * eg. if the url is "http://www.google.com/partners.html" this will return false, - * as the google keyword parameter couldn't be found. - * - * @see unit tests in /tests/core/Common.test.php - * @param string $referrerUrl URL referrer URL, eg. $_SERVER['HTTP_REFERER'] - * @return array|bool false if a keyword couldn't be extracted, - * or array( - * 'name' => 'Google', - * 'keywords' => 'my searched keywords') - */ - public static function extractSearchEngineInformationFromUrl($referrerUrl) - { - $referrerParsed = @parse_url($referrerUrl); - $referrerHost = ''; - if (isset($referrerParsed['host'])) { - $referrerHost = $referrerParsed['host']; - } - if (empty($referrerHost)) { - return false; - } - // some search engines (eg. Bing Images) use the same domain - // as an existing search engine (eg. Bing), we must also use the url path - $referrerPath = ''; - if (isset($referrerParsed['path'])) { - $referrerPath = $referrerParsed['path']; - } - - // no search query - if (!isset($referrerParsed['query'])) { - $referrerParsed['query'] = ''; - } - $query = $referrerParsed['query']; - - // Google Referrers URLs sometimes have the fragment which contains the keyword - if (!empty($referrerParsed['fragment'])) { - $query .= '&' . $referrerParsed['fragment']; - } - - $searchEngines = Common::getSearchEngineUrls(); - - $hostPattern = self::getLossyUrl($referrerHost); - /* - * Try to get the best matching 'host' in definitions - * 1. check if host + path matches an definition - * 2. check if host only matches - * 3. check if host pattern + path matches - * 4. check if host pattern matches - * 5. special handling - */ - if (array_key_exists($referrerHost . $referrerPath, $searchEngines)) { - $referrerHost = $referrerHost . $referrerPath; - } elseif (array_key_exists($referrerHost, $searchEngines)) { - // no need to change host - } elseif (array_key_exists($hostPattern . $referrerPath, $searchEngines)) { - $referrerHost = $hostPattern . $referrerPath; - } elseif (array_key_exists($hostPattern, $searchEngines)) { - $referrerHost = $hostPattern; - } elseif (!array_key_exists($referrerHost, $searchEngines)) { - if (!strncmp($query, 'cx=partner-pub-', 15)) { - // Google custom search engine - $referrerHost = 'google.com/cse'; - } elseif (!strncmp($referrerPath, '/pemonitorhosted/ws/results/', 28)) { - // private-label search powered by InfoSpace Metasearch - $referrerHost = 'wsdsold.infospace.com'; - } elseif (strpos($referrerHost, '.images.search.yahoo.com') != false) { - // Yahoo! Images - $referrerHost = 'images.search.yahoo.com'; - } elseif (strpos($referrerHost, '.search.yahoo.com') != false) { - // Yahoo! - $referrerHost = 'search.yahoo.com'; - } else { - return false; - } - } - $searchEngineName = $searchEngines[$referrerHost][0]; - $variableNames = null; - if (isset($searchEngines[$referrerHost][1])) { - $variableNames = $searchEngines[$referrerHost][1]; - } - if (!$variableNames) { - $searchEngineNames = Common::getSearchEngineNames(); - $url = $searchEngineNames[$searchEngineName]; - $variableNames = $searchEngines[$url][1]; - } - if (!is_array($variableNames)) { - $variableNames = array($variableNames); - } - - $key = null; - if ($searchEngineName === 'Google Images' - || ($searchEngineName === 'Google' && strpos($referrerUrl, '/imgres') !== false) - ) { - if (strpos($query, '&prev') !== false) { - $query = urldecode(trim(self::getParameterFromQueryString($query, 'prev'))); - $query = str_replace('&', '&', strstr($query, '?')); - } - $searchEngineName = 'Google Images'; - } elseif ($searchEngineName === 'Google' - && (strpos($query, '&as_') !== false || strpos($query, 'as_') === 0) - ) { - $keys = array(); - $key = self::getParameterFromQueryString($query, 'as_q'); - if (!empty($key)) { - array_push($keys, $key); - } - $key = self::getParameterFromQueryString($query, 'as_oq'); - if (!empty($key)) { - array_push($keys, str_replace('+', ' OR ', $key)); - } - $key = self::getParameterFromQueryString($query, 'as_epq'); - if (!empty($key)) { - array_push($keys, "\"$key\""); - } - $key = self::getParameterFromQueryString($query, 'as_eq'); - if (!empty($key)) { - array_push($keys, "-$key"); - } - $key = trim(urldecode(implode(' ', $keys))); - } - - if ($searchEngineName === 'Google') { - // top bar menu - $tbm = self::getParameterFromQueryString($query, 'tbm'); - switch ($tbm) { - case 'isch': - $searchEngineName = 'Google Images'; - break; - case 'vid': - $searchEngineName = 'Google Video'; - break; - case 'shop': - $searchEngineName = 'Google Shopping'; - break; - } - } - - if (empty($key)) { - foreach ($variableNames as $variableName) { - if ($variableName[0] == '/') { - // regular expression match - if (preg_match($variableName, $referrerUrl, $matches)) { - $key = trim(urldecode($matches[1])); - break; - } - } else { - // search for keywords now &vname=keyword - $key = self::getParameterFromQueryString($query, $variableName); - $key = trim(urldecode($key)); - - // Special cases: empty or no keywords - if (empty($key) - && ( - // Google search with no keyword - ($searchEngineName == 'Google' - && (empty($query) && (empty($referrerPath) || $referrerPath == '/') && empty($referrerParsed['fragment'])) - ) - - // Yahoo search with no keyword - || ($searchEngineName == 'Yahoo!' - && ($referrerParsed['host'] == 'r.search.yahoo.com') - ) - - // empty keyword parameter - || strpos($query, sprintf('&%s=', $variableName)) !== false - || strpos($query, sprintf('?%s=', $variableName)) !== false - - // search engines with no keyword - || $searchEngineName == 'Ixquick' - || $searchEngineName == 'Google Images' - || $searchEngineName == 'DuckDuckGo') - ) { - $key = false; - } - if (!empty($key) - || $key === false - ) { - break; - } - } - } - } - - // $key === false is the special case "No keyword provided" which is a Search engine match - if ($key === null - || $key === '' - ) { - return false; - } - - if (!empty($key)) { - if (function_exists('iconv') - && isset($searchEngines[$referrerHost][3]) - ) { - // accepts string, array, or comma-separated list string in preferred order - $charsets = $searchEngines[$referrerHost][3]; - if (!is_array($charsets)) { - $charsets = explode(',', $charsets); - } - - if (!empty($charsets)) { - $charset = $charsets[0]; - if (count($charsets) > 1 - && function_exists('mb_detect_encoding') - ) { - $charset = mb_detect_encoding($key, $charsets); - if ($charset === false) { - $charset = $charsets[0]; - } - } - - $newkey = @iconv($charset, 'UTF-8//IGNORE', $key); - if (!empty($newkey)) { - $key = $newkey; - } - } - } - - $key = Common::mb_strtolower($key); - } - - return array( - 'name' => $searchEngineName, - 'keywords' => $key, - ); - } - - /** * Returns the query part from any valid url and adds additional parameters to the query part if needed. * * @param string $url Any url eg `"http://example.com/piwik/?foo=bar"` diff --git a/plugins/CorePluginsAdmin/Marketplace.php b/plugins/CorePluginsAdmin/Marketplace.php index b5639ab621..b46d1932e4 100644 --- a/plugins/CorePluginsAdmin/Marketplace.php +++ b/plugins/CorePluginsAdmin/Marketplace.php @@ -114,14 +114,14 @@ class Marketplace $pluginsHavingUpdate = array(); } - foreach ($pluginsHavingUpdate as &$updatePlugin) { + foreach ($pluginsHavingUpdate as $key => $updatePlugin) { foreach ($loadedPlugins as $loadedPlugin) { if (!empty($updatePlugin['name']) && $loadedPlugin->getPluginName() == $updatePlugin['name'] ) { $updatePlugin['currentVersion'] = $loadedPlugin->getVersion(); $updatePlugin['isActivated'] = $pluginManager->isPluginActivated($updatePlugin['name']); - $updatePlugin = $this->addMissingRequirements($updatePlugin); + $pluginsHavingUpdate[$key] = $this->addMissingRequirements($updatePlugin); break; } } diff --git a/plugins/Referrers/API.php b/plugins/Referrers/API.php index d1c08ac20c..befd651fbe 100644 --- a/plugins/Referrers/API.php +++ b/plugins/Referrers/API.php @@ -230,14 +230,14 @@ class API extends \Piwik\Plugin\API $dataTable = Archive::createDataTableFromArchive(Archiver::SEARCH_ENGINES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded, $flat); if ($flat) { - $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'url', __NAMESPACE__ . '\getSearchEngineUrlFromName')); - $dataTable->filter('MetadataCallbackAddMetadata', array('url', 'logo', __NAMESPACE__ . '\getSearchEngineLogoFromUrl')); + $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'url', function ($url) { return SearchEngine::getInstance()->getUrlFromName($url); })); + $dataTable->filter('MetadataCallbackAddMetadata', array('url', 'logo', function ($url) { return SearchEngine::getInstance()->getLogoFromUrl($url); })); $dataTable->filterSubtables('Piwik\Plugins\Referrers\DataTable\Filter\KeywordsFromSearchEngineId', array($dataTable)); } else { $dataTable->filter('AddSegmentByLabel', array('referrerName')); $dataTable->queueFilter('PrependSegment', array('referrerType==search;')); - $dataTable->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', __NAMESPACE__ . '\getSearchEngineUrlFromName')); - $dataTable->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', __NAMESPACE__ . '\getSearchEngineLogoFromUrl')); + $dataTable->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', function ($url) { return SearchEngine::getInstance()->getUrlFromName($url); })); + $dataTable->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', function ($url) { return SearchEngine::getInstance()->getLogoFromUrl($url); })); } return $dataTable; @@ -309,9 +309,9 @@ class API extends \Piwik\Plugin\API { $dataTable = Archive::createDataTableFromArchive(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded, false); - $dataTable->filter('ColumnCallbackDeleteRow', array('label', function ($url) { return !isSocialUrl($url); })); - $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'url', __NAMESPACE__ . '\getSocialMainUrl')); - $dataTable->filter('GroupBy', array('label', __NAMESPACE__ . '\getSocialNetworkFromDomain')); + $dataTable->filter('ColumnCallbackDeleteRow', array('label', function ($url) { return !Social::getInstance()->isSocialUrl($url); })); + $dataTable->filter('ColumnCallbackAddMetadata', array('label', 'url', function ($url) { return Social::getInstance()->getMainUrl($url); })); + $dataTable->filter('GroupBy', array('label', function ($url) { return Social::getInstance()->getSocialNetworkFromDomain($url); })); $this->setSocialIdSubtables($dataTable); $this->removeSubtableMetadata($dataTable); @@ -320,7 +320,7 @@ class API extends \Piwik\Plugin\API $this->buildExpandedTableForFlattenGetSocials($idSite, $period, $date, $segment, $expanded, $dataTable); } - $dataTable->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', __NAMESPACE__ . '\getSocialsLogoFromUrl')); + $dataTable->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', function ($url) { return Social::getInstance()->getLogoFromUrl($url); })); return $dataTable; } @@ -334,7 +334,7 @@ class API extends \Piwik\Plugin\API * @param string $date * @param bool|string $segment * @param bool|int $idSubtable This ID does not reference a real DataTable record. Instead, it - * is the array index of an item in the /core/DataFiles/Socials.php file. + * is the array index of an item in the Socials list file. * The urls are filtered by the social network at this index. * If false, no filtering is done and every social URL is returned. * @return DataTable @@ -344,7 +344,7 @@ class API extends \Piwik\Plugin\API $dataTable = $this->getDataTable(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded = true); // get the social network domain referred to by $idSubtable - $socialNetworks = Common::getSocialUrls(); + $socialNetworks = Social::getInstance()->getDefinitions(); $social = false; if ($idSubtable !== false) { @@ -362,7 +362,7 @@ class API extends \Piwik\Plugin\API $dataTable->filter( 'ColumnCallbackDeleteRow', array('label', - function ($url) use ($social) { return !isSocialUrl($url, $social); } + function ($url) use ($social) { return !Social::getInstance()->isSocialUrl($url, $social); } ) ); @@ -428,7 +428,7 @@ class API extends \Piwik\Plugin\API /** * Sets the subtable IDs for the DataTable returned by getSocial. * - * The IDs are int indexes into the array in /core/DataFiles/Socials.php. + * The IDs are int indexes into the array in of defined socials. * * @param DataTable $dataTable */ @@ -443,7 +443,7 @@ class API extends \Piwik\Plugin\API $socialName = $row->getColumn('label'); $i = 1; // start at one because idSubtable=0 is equivalent to idSubtable=false - foreach (Common::getSocialUrls() as $name) { + foreach (Social::getInstance()->getDefinitions() as $name) { if ($name == $socialName) { $row->setNonLoadedSubtableId($i); break; @@ -491,7 +491,7 @@ class API extends \Piwik\Plugin\API { $urlsTable = Archive::createDataTableFromArchive(Archiver::WEBSITES_RECORD_NAME, $idSite, $period, $date, $segment, $expanded, $flat = true); $urlsTable->filter('ColumnCallbackDeleteRow', array('label', function ($url) { - return !isSocialUrl($url); + return !Social::getInstance()->isSocialUrl($url); })); $urlsTable = $urlsTable->mergeSubtables(); @@ -504,7 +504,7 @@ class API extends \Piwik\Plugin\API $rows = $urlsTable->getRows(); foreach ($rows as $id => $urlsTableRow) { $url = $urlsTableRow->getColumn('label'); - if (isSocialUrl($url, $social)) { + if (Social::getInstance()->isSocialUrl($url, $social)) { $newTable->addRow($urlsTableRow); $urlsTable->deleteRow($id); } diff --git a/plugins/Referrers/Columns/Base.php b/plugins/Referrers/Columns/Base.php index 78fe27516c..df86b60e65 100644 --- a/plugins/Referrers/Columns/Base.php +++ b/plugins/Referrers/Columns/Base.php @@ -11,6 +11,7 @@ namespace Piwik\Plugins\Referrers\Columns; use Piwik\Common; use Piwik\Piwik; use Piwik\Plugin\Dimension\VisitDimension; +use Piwik\Plugins\Referrers\SearchEngine AS SearchEngineDetection; use Piwik\Tracker\PageUrl; use Piwik\Tracker\Request; use Piwik\Tracker\Visit; @@ -139,7 +140,7 @@ abstract class Base extends VisitDimension */ protected function detectReferrerSearchEngine() { - $searchEngineInformation = UrlHelper::extractSearchEngineInformationFromUrl($this->referrerUrl); + $searchEngineInformation = SearchEngineDetection::getInstance()->extractInformationFromUrl($this->referrerUrl); /** * Triggered when detecting the search engine of a referrer URL. @@ -277,7 +278,7 @@ abstract class Base extends VisitDimension // Set the Campaign keyword to the keyword found in the Referrer URL if any if (!empty($this->nameReferrerAnalyzed)) { - $referrerUrlInfo = UrlHelper::extractSearchEngineInformationFromUrl($this->referrerUrl); + $referrerUrlInfo = SearchEngineDetection::getInstance()->extractInformationFromUrl($this->referrerUrl); if (!empty($referrerUrlInfo['keywords'])) { $this->keywordReferrerAnalyzed = $referrerUrlInfo['keywords']; } diff --git a/plugins/Referrers/DataTable/Filter/KeywordsFromSearchEngineId.php b/plugins/Referrers/DataTable/Filter/KeywordsFromSearchEngineId.php index 1688713a86..789a1b1a6a 100644 --- a/plugins/Referrers/DataTable/Filter/KeywordsFromSearchEngineId.php +++ b/plugins/Referrers/DataTable/Filter/KeywordsFromSearchEngineId.php @@ -11,6 +11,7 @@ namespace Piwik\Plugins\Referrers\DataTable\Filter; use Piwik\DataTable\BaseFilter; use Piwik\DataTable\Row; use Piwik\DataTable; +use Piwik\Plugins\Referrers\SearchEngine; class KeywordsFromSearchEngineId extends BaseFilter { @@ -47,7 +48,7 @@ class KeywordsFromSearchEngineId extends BaseFilter if (!empty($subTableRow)) { $searchEngineUrl = $subTableRow->getMetadata('url'); - $table->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', 'Piwik\Plugins\Referrers\getSearchEngineUrlFromKeywordAndUrl', array($searchEngineUrl))); + $table->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', function ($keyword, $url) { return SearchEngine::getInstance()->getBackLinkFromUrlAndKeyword($url, $keyword); }, array($searchEngineUrl))); $table->queueFilter(function (DataTable $table) { $row = $table->getRowFromId(DataTable::ID_SUMMARY_ROW); if ($row) { diff --git a/plugins/Referrers/DataTable/Filter/SearchEnginesFromKeywordId.php b/plugins/Referrers/DataTable/Filter/SearchEnginesFromKeywordId.php index feab8636af..6d395abba0 100644 --- a/plugins/Referrers/DataTable/Filter/SearchEnginesFromKeywordId.php +++ b/plugins/Referrers/DataTable/Filter/SearchEnginesFromKeywordId.php @@ -11,6 +11,7 @@ namespace Piwik\Plugins\Referrers\DataTable\Filter; use Piwik\DataTable\BaseFilter; use Piwik\DataTable\Row; use Piwik\DataTable; +use Piwik\Plugins\Referrers\SearchEngine; class SearchEnginesFromKeywordId extends BaseFilter { @@ -44,14 +45,14 @@ class SearchEnginesFromKeywordId extends BaseFilter { $idSubtable = $this->idSubtable ? : $table->getId(); - $table->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', 'Piwik\Plugins\Referrers\getSearchEngineUrlFromName')); - $table->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', 'Piwik\Plugins\Referrers\getSearchEngineLogoFromUrl')); + $table->queueFilter('ColumnCallbackAddMetadata', array('label', 'url', function ($url) { return SearchEngine::getInstance()->getUrlFromName($url); })); + $table->queueFilter('MetadataCallbackAddMetadata', array('url', 'logo', function ($url) { return SearchEngine::getInstance()->getLogoFromUrl($url); })); // get the keyword and create the URL to the search result page $rootRow = $this->firstLevelKeywordTable->getRowFromIdSubDataTable($idSubtable); if ($rootRow) { $keyword = $rootRow->getColumn('label'); - $table->queueFilter('MetadataCallbackReplace', array('url', 'Piwik\Plugins\Referrers\getSearchEngineUrlFromUrlAndKeyword', array($keyword))); + $table->queueFilter('MetadataCallbackReplace', array('url', function ($url, $keyword) { return SearchEngine::getInstance()->getBackLinkFromUrlAndKeyword($url, $keyword); }, array($keyword))); $table->queueFilter(function (DataTable $table) { $row = $table->getRowFromId(DataTable::ID_SUMMARY_ROW); if ($row) { diff --git a/plugins/Referrers/SearchEngine.php b/plugins/Referrers/SearchEngine.php new file mode 100644 index 0000000000..14009a9029 --- /dev/null +++ b/plugins/Referrers/SearchEngine.php @@ -0,0 +1,485 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ +namespace Piwik\Plugins\Referrers; + +use Piwik\Cache; +use Piwik\Common; +use Piwik\Option; +use Piwik\Piwik; +use Piwik\Singleton; +use Piwik\UrlHelper; + +/** + * Contains methods to access search engine definition data. + */ +class SearchEngine extends Singleton +{ + const OPTION_STORAGE_NAME = 'SearchEngineDefinitions'; + + /** @var string location of definition file (relative to PIWIK_INCLUDE_PATH) */ + const DEFINITION_FILE = '/vendor/piwik/searchengine-and-social-list/SearchEngines.yml'; + + protected $definitionList = null; + + /** + * Returns list of search engines by URL + * + * @return array Array of ( URL => array( searchEngineName, keywordParameter, path, charset ) ) + */ + public function getDefinitions() + { + $cache = Cache::getEagerCache(); + $cacheId = 'SearchEngine-' . self::OPTION_STORAGE_NAME; + + if ($cache->contains($cacheId)) { + $list = $cache->fetch($cacheId); + } else { + $list = $this->loadDefinitions(); + $cache->save($cacheId, $list); + } + + return $list; + } + + private function loadDefinitions() + { + if (empty($this->definitionList)) { + // Read first from the auto-updated list in database + $list = Option::get(self::OPTION_STORAGE_NAME); + + if ($list) { + $this->definitionList = unserialize(base64_decode($list)); + } else { + // Fallback to reading the bundled list + $yml = file_get_contents(PIWIK_INCLUDE_PATH . self::DEFINITION_FILE); + $this->definitionList = $this->loadYmlData($yml); + Option::set(self::OPTION_STORAGE_NAME, base64_encode(serialize($this->definitionList))); + } + } + + Piwik::postEvent('Referrer.addSearchEngineUrls', array(&$this->definitionList)); + + $this->convertLegacyDefinitions(); + + return $this->definitionList; + } + + /** + * @deprecated remove in 3.0 + */ + protected function convertLegacyDefinitions() + { + foreach ($this->definitionList as $url => $definition) { + if (!array_key_exists('name', $definition) && isset($definition[0]) && isset($definition[1])) { + $this->definitionList[$url] = array( + 'name' => $definition[0], + 'params' => $definition[1], + 'backlink' => @$definition[2], + 'charsets' => @$definition[3] + ); + } + } + + } + + /** + * Parses the given YML string and caches the resulting definitions + * + * @param string $yml + * @return array + */ + public function loadYmlData($yml) + { + $searchEngines = \Spyc::YAMLLoadString($yml); + + $this->definitionList = $this->transformData($searchEngines); + + return $this->definitionList; + } + + protected function transformData($searchEngines) + { + $urlToInfo = array(); + foreach ($searchEngines as $name => $info) { + foreach ($info as $urlDefinitions) { + foreach ($urlDefinitions['urls'] as $url) { + $searchEngineData = $urlDefinitions; + unset($searchEngineData['urls']); + $searchEngineData['name'] = $name; + $urlToInfo[$url] = $searchEngineData; + } + } + } + return $urlToInfo; + } + + /** + * Returns list of search engines by name + * + * @return array Array of ( searchEngineName => URL ) + */ + public function getNames() + { + $cacheId = 'SearchEngine.getSearchEngineNames'; + $cache = Cache::getTransientCache(); + $nameToUrl = $cache->fetch($cacheId); + + if (empty($nameToUrl)) { + $searchEngines = $this->getDefinitions(); + + $nameToUrl = array(); + foreach ($searchEngines as $url => $info) { + if (!isset($nameToUrl[$info['name']])) { + $nameToUrl[$info['name']] = $url; + } + } + $cache->save($cacheId, $nameToUrl); + } + + return $nameToUrl; + } + + /** + * Returns definitions for the given search engine host + * + * @param string $host + * @return array + */ + public function getDefinitionByHost($host) + { + $searchEngines = $this->getDefinitions(); + + if (!array_key_exists($host, $searchEngines)) { + return array(); + } + + return $searchEngines[$host]; + } + + /** + * Extracts a keyword from a raw not encoded URL. + * Will only extract keyword if a known search engine has been detected. + * Returns the keyword: + * - in UTF8: automatically converted from other charsets when applicable + * - strtolowered: "QUErY test!" will return "query test!" + * - trimmed: extra spaces before and after are removed + * + * The function returns false when a keyword couldn't be found. + * eg. if the url is "http://www.google.com/partners.html" this will return false, + * as the google keyword parameter couldn't be found. + * + * @see unit tests in /tests/core/Common.test.php + * @param string $referrerUrl URL referrer URL, eg. $_SERVER['HTTP_REFERER'] + * @return array|bool false if a keyword couldn't be extracted, + * or array( + * 'name' => 'Google', + * 'keywords' => 'my searched keywords') + */ + public function extractInformationFromUrl($referrerUrl) + { + $referrerParsed = @parse_url($referrerUrl); + $referrerHost = ''; + if (isset($referrerParsed['host'])) { + $referrerHost = $referrerParsed['host']; + } + if (empty($referrerHost)) { + return false; + } + // some search engines (eg. Bing Images) use the same domain + // as an existing search engine (eg. Bing), we must also use the url path + $referrerPath = ''; + if (isset($referrerParsed['path'])) { + $referrerPath = $referrerParsed['path']; + } + + $query = ''; + if (isset($referrerParsed['query'])) { + $query = $referrerParsed['query']; + } + + // Google Referrers URLs sometimes have the fragment which contains the keyword + if (!empty($referrerParsed['fragment'])) { + $query .= '&' . $referrerParsed['fragment']; + } + + $referrerHost = $this->getEngineHostFromUrl($referrerHost, $referrerPath, $query); + + if (empty($referrerHost)) { + return false; + } + + $definitions = $this->getDefinitionByHost($referrerHost); + + $searchEngineName = $definitions['name']; + $variableNames = $definitions['params']; + + $key = null; + if ($searchEngineName === 'Google Images' + || ($searchEngineName === 'Google' && strpos($referrerUrl, '/imgres') !== false) + ) { + if (strpos($query, '&prev') !== false) { + $query = urldecode(trim(UrlHelper::getParameterFromQueryString($query, 'prev'))); + $query = str_replace('&', '&', strstr($query, '?')); + } + $searchEngineName = 'Google Images'; + } elseif ($searchEngineName === 'Google' + && (strpos($query, '&as_') !== false || strpos($query, 'as_') === 0) + ) { + $keys = array(); + $key = UrlHelper::getParameterFromQueryString($query, 'as_q'); + if (!empty($key)) { + array_push($keys, $key); + } + $key = UrlHelper::getParameterFromQueryString($query, 'as_oq'); + if (!empty($key)) { + array_push($keys, str_replace('+', ' OR ', $key)); + } + $key = UrlHelper::getParameterFromQueryString($query, 'as_epq'); + if (!empty($key)) { + array_push($keys, "\"$key\""); + } + $key = UrlHelper::getParameterFromQueryString($query, 'as_eq'); + if (!empty($key)) { + array_push($keys, "-$key"); + } + $key = trim(urldecode(implode(' ', $keys))); + } + + if ($searchEngineName === 'Google') { + // top bar menu + $tbm = UrlHelper::getParameterFromQueryString($query, 'tbm'); + switch ($tbm) { + case 'isch': + $searchEngineName = 'Google Images'; + break; + case 'vid': + $searchEngineName = 'Google Video'; + break; + case 'shop': + $searchEngineName = 'Google Shopping'; + break; + } + } + + if (empty($key)) { + foreach ($variableNames as $variableName) { + if ($variableName[0] == '/') { + // regular expression match + if (preg_match($variableName, $referrerUrl, $matches)) { + $key = trim(urldecode($matches[1])); + break; + } + } else { + // search for keywords now &vname=keyword + $key = UrlHelper::getParameterFromQueryString($query, $variableName); + $key = trim(urldecode($key)); + + // Special cases: empty or no keywords + if (empty($key) + && ( + // Google search with no keyword + ($searchEngineName == 'Google' + && (empty($query) && (empty($referrerPath) || $referrerPath == '/') && empty($referrerParsed['fragment'])) + ) + + // Yahoo search with no keyword + || ($searchEngineName == 'Yahoo!' + && ($referrerParsed['host'] == 'r.search.yahoo.com') + ) + + // empty keyword parameter + || strpos($query, sprintf('&%s=', $variableName)) !== false + || strpos($query, sprintf('?%s=', $variableName)) !== false + + // search engines with no keyword + || $searchEngineName == 'Ixquick' + || $searchEngineName == 'Google Images' + || $searchEngineName == 'DuckDuckGo') + ) { + $key = false; + } + if (!empty($key) + || $key === false + ) { + break; + } + } + } + } + + // $key === false is the special case "No keyword provided" which is a Search engine match + if ($key === null || $key === '') { + return false; + } + + if (!empty($key)) { + if (!empty($definitions['charsets'])) { + $key = $this->convertCharset($key, $definitions['charsets']); + } + $key = Common::mb_strtolower($key); + } + + return array( + 'name' => $searchEngineName, + 'keywords' => $key, + ); + } + + protected function getEngineHostFromUrl($host, $path, $query) + { + $searchEngines = $this->getDefinitions(); + + $hostPattern = UrlHelper::getLossyUrl($host); + /* + * Try to get the best matching 'host' in definitions + * 1. check if host + path matches an definition + * 2. check if host only matches + * 3. check if host pattern + path matches + * 4. check if host pattern matches + * 5. special handling + */ + if (array_key_exists($host . $path, $searchEngines)) { + $host = $host . $path; + } elseif (array_key_exists($host, $searchEngines)) { + // no need to change host + } elseif (array_key_exists($hostPattern . $path, $searchEngines)) { + $host = $hostPattern . $path; + } elseif (array_key_exists($hostPattern, $searchEngines)) { + $host = $hostPattern; + } elseif (!array_key_exists($host, $searchEngines)) { + if (!strncmp($query, 'cx=partner-pub-', 15)) { + // Google custom search engine + $host = 'google.com/cse'; + } elseif (!strncmp($path, '/pemonitorhosted/ws/results/', 28)) { + // private-label search powered by InfoSpace Metasearch + $host = 'wsdsold.infospace.com'; + } elseif (strpos($host, '.images.search.yahoo.com') != false) { + // Yahoo! Images + $host = 'images.search.yahoo.com'; + } elseif (strpos($host, '.search.yahoo.com') != false) { + // Yahoo! + $host = 'search.yahoo.com'; + } else { + return false; + } + } + + return $host; + } + + /** + * Tries to convert the given string from one of the given charsets to UTF-8 + * @param string $string + * @param array $charsets + * @return string + */ + protected function convertCharset($string, $charsets) + { + if (function_exists('iconv') + && !empty($charsets) + ) { + $charset = $charsets[0]; + if (count($charsets) > 1 + && function_exists('mb_detect_encoding') + ) { + $charset = mb_detect_encoding($string, $charsets); + if ($charset === false) { + $charset = $charsets[0]; + } + } + + $newKey = @iconv($charset, 'UTF-8//IGNORE', $string); + if (!empty($newKey)) { + $string = $newKey; + } + } + + return $string; + } + + /** + * Return search engine URL by name + * + * @see core/DataFiles/SearchEnginges.php + * + * @param string $name + * @return string URL + */ + public function getUrlFromName($name) + { + $searchEngineNames = $this->getNames(); + if (isset($searchEngineNames[$name])) { + $url = 'http://' . $searchEngineNames[$name]; + } else { + $url = 'URL unknown!'; + } + return $url; + } + + /** + * Return search engine host in URL + * + * @param string $url + * @return string host + */ + private function getHostFromUrl($url) + { + if (strpos($url, '//')) { + $url = substr($url, strpos($url, '//') + 2); + } + if (($p = strpos($url, '/')) !== false) { + $url = substr($url, 0, $p); + } + return $url; + } + + + /** + * Return search engine logo path by URL + * + * @param string $url + * @return string path + * @see plugins/Referrers/images/searchEnginges/ + */ + public function getLogoFromUrl($url) + { + $pathInPiwik = 'plugins/Referrers/images/searchEngines/%s.png'; + $pathWithCode = sprintf($pathInPiwik, $this->getHostFromUrl($url)); + $absolutePath = PIWIK_INCLUDE_PATH . '/' . $pathWithCode; + if (file_exists($absolutePath)) { + return $pathWithCode; + } + return sprintf($pathInPiwik, 'xx'); + } + + /** + * Return search engine URL for URL and keyword + * + * @see core/DataFiles/SearchEnginges.php + * + * @param string $url Domain name, e.g., search.piwik.org + * @param string $keyword Keyword, e.g., web+analytics + * @return string URL, e.g., http://search.piwik.org/q=web+analytics + */ + public function getBackLinkFromUrlAndKeyword($url, $keyword) + { + if ($keyword === API::LABEL_KEYWORD_NOT_DEFINED) { + return 'http://piwik.org/faq/general/#faq_144'; + } + $keyword = urlencode($keyword); + $keyword = str_replace(urlencode('+'), urlencode(' '), $keyword); + $host = substr($url, strpos($url, '//') + 2); + $definition = $this->getDefinitionByHost($host); + if (empty($definition['backlink'])) { + return false; + } + $path = str_replace("{k}", $keyword, $definition['backlink']); + return $url . (substr($url, -1) != '/' ? '/' : '') . $path; + } +} diff --git a/plugins/Referrers/Social.php b/plugins/Referrers/Social.php new file mode 100644 index 0000000000..71badc3357 --- /dev/null +++ b/plugins/Referrers/Social.php @@ -0,0 +1,181 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ +namespace Piwik\Plugins\Referrers; +use Piwik\Cache; +use Piwik\Common; +use Piwik\Option; +use Piwik\Piwik; +use Piwik\Singleton; +use Piwik\UrlHelper; + +/** + * Contains methods to access search engine definition data. + */ +class Social extends Singleton +{ + const OPTION_STORAGE_NAME = 'SocialDefinitions'; + + /** @var string location of definition file (relative to PIWIK_INCLUDE_PATH) */ + const DEFINITION_FILE = '/vendor/piwik/searchengine-and-social-list/Socials.yml'; + + protected $definitionList = null; + + /** + * Returns list of search engines by URL + * + * @return array Array of ( URL => array( searchEngineName, keywordParameter, path, charset ) ) + */ + public function getDefinitions() + { + $cache = Cache::getEagerCache(); + $cacheId = 'Social-' . self::OPTION_STORAGE_NAME; + + if ($cache->contains($cacheId)) { + $list = $cache->fetch($cacheId); + } else { + $list = $this->loadDefinitions(); + $cache->save($cacheId, $list); + } + + return $list; + } + + private function loadDefinitions() + { + if ($this->definitionList === null) { + // Read first from the auto-updated list in database + $list = Option::get(self::OPTION_STORAGE_NAME); + + if ($list) { + $this->definitionList = unserialize(base64_decode($list)); + } else { + // Fallback to reading the bundled list + $yml = file_get_contents(PIWIK_INCLUDE_PATH . self::DEFINITION_FILE); + $this->definitionList = $this->loadYmlData($yml); + Option::set(self::OPTION_STORAGE_NAME, base64_encode(serialize($this->definitionList))); + } + } + + Piwik::postEvent('Referrer.addSocialUrls', array(&$this->definitionList)); + + return $this->definitionList; + } + + /** + * Parses the given YML string and caches the resulting definitions + * + * @param string $yml + * @return array + */ + public function loadYmlData($yml) + { + $searchEngines = \Spyc::YAMLLoadString($yml); + + $this->definitionList = $this->transformData($searchEngines); + + return $this->definitionList; + } + + protected function transformData($socials) + { + $urlToName = array(); + foreach ($socials as $name => $urls) { + foreach ($urls as $url) { + $urlToName[$url] = $name; + } + } + return $urlToName; + } + + /** + * Returns true if a URL belongs to a social network, false if otherwise. + * + * @param string $url The URL to check. + * @param string|bool $socialName The social network's name to check for, or false to check + * for any. + * @return bool + */ + public function isSocialUrl($url, $socialName = false) + { + foreach ($this->getDefinitions() as $domain => $name) { + + if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url) && ($socialName === false || $name == $socialName)) { + + return true; + } + } + + return false; + } + + + /** + * Get's social network name from URL. + * + * @param string $url + * @return string + */ + public function getSocialNetworkFromDomain($url) + { + foreach ($this->getDefinitions() as $domain => $name) { + + if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url)) { + + return $name; + } + } + + return Piwik::translate('General_Unknown'); + } + + /** + * Returns the main url of the social network the given url matches + * + * @param string $url + * + * @return string + */ + public function getMainUrl($url) + { + $social = $this->getSocialNetworkFromDomain($url); + foreach ($this->getDefinitions() as $domain => $name) { + + if ($name == $social) { + + return $domain; + } + } + return $url; + } + + + /** + * Return social network logo path by URL + * + * @param string $domain + * @return string path + * @see plugins/Referrers/images/socials/ + */ + public function getLogoFromUrl($domain) + { + $social = $this->getSocialNetworkFromDomain($domain); + $socialNetworks = $this->getDefinitions(); + + $filePattern = 'plugins/Referrers/images/socials/%s.png'; + + $socialDomains = array_keys($socialNetworks, $social); + foreach ($socialDomains as $domain) { + if (file_exists(PIWIK_INCLUDE_PATH . '/' . sprintf($filePattern, $domain))) { + return sprintf($filePattern, $domain); + } + } + + return sprintf($filePattern, 'xx'); + } +} diff --git a/plugins/Referrers/Tasks.php b/plugins/Referrers/Tasks.php new file mode 100644 index 0000000000..da9cad1321 --- /dev/null +++ b/plugins/Referrers/Tasks.php @@ -0,0 +1,54 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + * + */ +namespace Piwik\Plugins\Referrers; + + +use Piwik\Http; +use Piwik\Option; + +class Tasks extends \Piwik\Plugin\Tasks +{ + public function schedule() + { + $this->weekly('updateSearchEngines'); + $this->weekly('updateSocials'); + } + + /** + * Update the search engine definitions + * + * @see https://github.com/piwik/searchengine-and-social-list + */ + public function updateSearchEngines() + { + $url = 'https://raw.githubusercontent.com/piwik/searchengine-and-social-list/master/SearchEngines.yml'; + $list = Http::sendHttpRequest($url, 30); + $searchEngines = SearchEngine::getInstance()->loadYmlData($list); + if (count($searchEngines) < 200) { + return; + } + Option::set(SearchEngine::OPTION_STORAGE_NAME, base64_encode(serialize($searchEngines))); + } + + /** + * Update the social definitions + * + * @see https://github.com/piwik/searchengine-and-social-list + */ + public function updateSocials() + { + $url = 'https://raw.githubusercontent.com/piwik/searchengine-and-social-list/master/Socials.yml'; + $list = Http::sendHttpRequest($url, 30); + $socials = Social::getInstance()->loadYmlData($list); + if (count($socials) < 50) { + return; + } + Option::set(Social::OPTION_STORAGE_NAME, base64_encode(serialize($socials))); + } +}
\ No newline at end of file diff --git a/plugins/Referrers/Visitor.php b/plugins/Referrers/Visitor.php index 308c7b08f8..b10b9b953d 100644 --- a/plugins/Referrers/Visitor.php +++ b/plugins/Referrers/Visitor.php @@ -62,7 +62,7 @@ class Visitor ) { $refUrl = @parse_url($this->details['referer_url']); if (isset($refUrl['host'])) { - $url = getSearchEngineUrlFromUrlAndKeyword('http://google.com', $this->getKeyword()); + $url = SearchEngine::getInstance()->getBackLinkFromUrlAndKeyword('http://google.com', $this->getKeyword()); $url = str_replace('google.com', $refUrl['host'], $url); return $url; @@ -109,7 +109,7 @@ class Visitor && !empty($this->details['referer_name']) ) { - return getSearchEngineUrlFromName($this->details['referer_name']); + return SearchEngine::getInstance()->getUrlFromName($this->details['referer_name']); } return null; @@ -121,7 +121,7 @@ class Visitor if (!is_null($searchEngineUrl)) { - return getSearchEngineLogoFromUrl($searchEngineUrl); + return SearchEngine::getInstance()->getLogoFromUrl($searchEngineUrl); } return null; diff --git a/plugins/Referrers/functions.php b/plugins/Referrers/functions.php index e0fee30833..c91ebc816d 100644 --- a/plugins/Referrers/functions.php +++ b/plugins/Referrers/functions.php @@ -28,194 +28,6 @@ function getPathFromUrl($url) } /** - * Returns the main url of the social network the given url matches - * - * @param string $url - * - * @return string - */ -function getSocialMainUrl($url) -{ - $social = getSocialNetworkFromDomain($url); - foreach (Common::getSocialUrls() as $domain => $name) { - - if ($name == $social) { - - return $domain; - } - } - return $url; -} - -/** - * Get's social network name from URL. - * - * @param string $url - * @return string - */ -function getSocialNetworkFromDomain($url) -{ - foreach (Common::getSocialUrls() as $domain => $name) { - - if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url)) { - - return $name; - } - } - - return Piwik::translate('General_Unknown'); -} - -/** - * Returns true if a URL belongs to a social network, false if otherwise. - * - * @param string $url The URL to check. - * @param string|bool $socialName The social network's name to check for, or false to check - * for any. - * @return bool - */ -function isSocialUrl($url, $socialName = false) -{ - foreach (Common::getSocialUrls() as $domain => $name) { - - if (preg_match('/(^|[\.\/])'.$domain.'([\.\/]|$)/', $url) && ($socialName === false || $name == $socialName)) { - - return true; - } - } - - return false; -} - -/** - * Return social network logo path by URL - * - * @param string $domain - * @return string path - * @see plugins/Referrers/images/socials/ - */ -function getSocialsLogoFromUrl($domain) -{ - $social = getSocialNetworkFromDomain($domain); - $socialNetworks = Common::getSocialUrls(); - - $filePattern = 'plugins/Referrers/images/socials/%s.png'; - - foreach ($socialNetworks as $domainKey => $name) { - if ($social == $socialNetworks[$domainKey] && file_exists(PIWIK_INCLUDE_PATH . '/' . sprintf($filePattern, $domainKey))) { - return sprintf($filePattern, $domainKey); - } - } - - return sprintf($filePattern, 'xx'); -} - -/** - * Return search engine URL by name - * - * @see core/DataFiles/SearchEnginges.php - * - * @param string $name - * @return string URL - */ -function getSearchEngineUrlFromName($name) -{ - $searchEngineNames = Common::getSearchEngineNames(); - if (isset($searchEngineNames[$name])) { - $url = 'http://' . $searchEngineNames[$name]; - } else { - $url = 'URL unknown!'; - } - return $url; -} - -/** - * Return search engine host in URL - * - * @param string $url - * @return string host - */ -function getSearchEngineHostFromUrl($url) -{ - if (strpos($url, '//')) { - $url = substr($url, strpos($url, '//') + 2); - } - if (($p = strpos($url, '/')) !== false) { - $url = substr($url, 0, $p); - } - return $url; -} - -/** - * Return search engine logo path by URL - * - * @param string $url - * @return string path - * @see plugins/Referrers/images/searchEnginges/ - */ -function getSearchEngineLogoFromUrl($url) -{ - $pathInPiwik = 'plugins/Referrers/images/searchEngines/%s.png'; - $pathWithCode = sprintf($pathInPiwik, getSearchEngineHostFromUrl($url)); - $absolutePath = PIWIK_INCLUDE_PATH . '/' . $pathWithCode; - if (file_exists($absolutePath)) { - return $pathWithCode; - } - return sprintf($pathInPiwik, 'xx'); -} - -/** - * Return search engine host and path in URL - * - * @param string $url - * @return string host - */ -function getSearchEngineHostPathFromUrl($url) -{ - $url = substr($url, strpos($url, '//') + 2); - return $url; -} - -/** - * Return search engine URL for URL and keyword - * - * @see core/DataFiles/SearchEnginges.php - * - * @param string $url Domain name, e.g., search.piwik.org - * @param string $keyword Keyword, e.g., web+analytics - * @return string URL, e.g., http://search.piwik.org/q=web+analytics - */ -function getSearchEngineUrlFromUrlAndKeyword($url, $keyword) -{ - if ($keyword === API::LABEL_KEYWORD_NOT_DEFINED) { - return 'http://piwik.org/faq/general/#faq_144'; - } - $searchEngineUrls = Common::getSearchEngineUrls(); - $keyword = urlencode($keyword); - $keyword = str_replace(urlencode('+'), urlencode(' '), $keyword); - $path = @$searchEngineUrls[getSearchEngineHostPathFromUrl($url)][2]; - if (empty($path)) { - return false; - } - $path = str_replace("{k}", $keyword, $path); - return $url . (substr($url, -1) != '/' ? '/' : '') . $path; -} - -/** - * Return search engine URL for keyword and URL - * - * @see \Piwik\Plugins\Referrers\getSearchEngineUrlFromUrlAndKeyword - * - * @param string $keyword Keyword, e.g., web+analytics - * @param string $url Domain name, e.g., search.piwik.org - * @return string URL, e.g., http://search.piwik.org/q=web+analytics - */ -function getSearchEngineUrlFromKeywordAndUrl($keyword, $url) -{ - return getSearchEngineUrlFromUrlAndKeyword($url, $keyword); -} - -/** * Return translated referrer type * * @param string $label @@ -223,7 +35,6 @@ function getSearchEngineUrlFromKeywordAndUrl($keyword, $url) */ function getReferrerTypeLabel($label) { - $indexTranslation = ''; switch ($label) { case Common::REFERRER_TYPE_DIRECT_ENTRY: $indexTranslation = 'Referrers_DirectEntry'; diff --git a/plugins/Referrers/tests/Unit/ReferrersTest.php b/plugins/Referrers/tests/Unit/ReferrersTest.php index 909e6bf65e..e5dea866e9 100644 --- a/plugins/Referrers/tests/Unit/ReferrersTest.php +++ b/plugins/Referrers/tests/Unit/ReferrersTest.php @@ -11,217 +11,15 @@ namespace Piwik\Plugins\Referrers\tests; use Piwik\DataTable; use Piwik\DataTable\Row; use Piwik\Period; +use Piwik\Plugins\Referrers\SearchEngine; require_once PIWIK_INCLUDE_PATH . '/plugins/Referrers/Referrers.php'; +/** + * @group Plugin + */ class ReferrersTest extends \PHPUnit_Framework_TestCase { - /** - * Dataprovider serving all search engine data - */ - public function getSearchEngines() - { - include PIWIK_PATH_TEST_TO_ROOT . '/core/DataFiles/SearchEngines.php'; - - $searchEngines = array(); - foreach ($GLOBALS['Piwik_SearchEngines'] as $url => $searchEngine) { - $searchEngines[] = array($url, $searchEngine); - } - return $searchEngines; - } - - /** - * search engine has at least one keyword - * - * @group Plugins - * - * @dataProvider getSearchEngines - */ - public function testMissingSearchEngineKeyword($url, $searchEngine) - { - // Get list of search engines and first appearing URL - static $searchEngines = array(); - - $name = parse_url('http://' . $url); - if (!array_key_exists($searchEngine[0], $searchEngines)) { - $searchEngines[$searchEngine[0]] = $url; - - $this->assertTrue(!empty($searchEngine[1]), $name['host']); - } - } - - /** - * search engine is defined in DataFiles/SearchEngines.php but there's no favicon - * - * @group Plugins - * - * @dataProvider getSearchEngines - */ - public function testMissingSearchEngineIcons($url, $searchEngine) - { - // Get list of existing favicons - $favicons = scandir(PIWIK_PATH_TEST_TO_ROOT . '/plugins/Referrers/images/searchEngines/'); - - // Get list of search engines and first appearing URL - static $searchEngines = array(); - - $name = parse_url('http://' . $url); - if (!array_key_exists($searchEngine[0], $searchEngines)) { - $searchEngines[$searchEngine[0]] = $url; - - $this->assertTrue(in_array($name['host'] . '.png', $favicons), $name['host']); - } - } - - /** - * favicon exists but there's no corresponding search engine defined in DataFiles/SearchEngines.php - * - * @group Plugins - */ - public function testObsoleteSearchEngineIcons() - { - include PIWIK_PATH_TEST_TO_ROOT . '/core/DataFiles/SearchEngines.php'; - - // Get list of search engines and first appearing URL - $searchEngines = array(); - foreach ($GLOBALS['Piwik_SearchEngines'] as $url => $searchEngine) { - $name = parse_url('http://' . $url); - if (!array_key_exists($name['host'], $searchEngines)) { - $searchEngines[$name['host']] = true; - } - } - - // Get list of existing favicons - $favicons = scandir(PIWIK_PATH_TEST_TO_ROOT . '/plugins/Referrers/images/searchEngines/'); - foreach ($favicons as $name) { - if ($name[0] == '.' || strpos($name, 'xx.') === 0) { - continue; - } - - $host = substr($name, 0, -4); - $this->assertTrue(array_key_exists($host, $searchEngines), $host); - } - } - - /** - * get search engine host from url - * - * @group Plugins - */ - public function testGetSearchEngineHostFromUrl() - { - $data = array( - 'http://www.google.com/cse' => array('www.google.com', 'www.google.com/cse'), - 'http://www.google.com' => array('www.google.com', 'www.google.com'), - ); - - foreach ($data as $url => $expected) { - $this->assertEquals($expected[0], \Piwik\Plugins\Referrers\getSearchEngineHostFromUrl($url)); - $this->assertEquals($expected[1], \Piwik\Plugins\Referrers\getSearchEngineHostPathFromUrl($url)); - } - } - - /** - * Dataprovider for testGetSearchEngineUrlFromUrlAndKeyword - */ - public function getSearchEngineUrlFromUrlAndKeywordTestData() - { - return array( - array('http://apollo.lv/portal/search/', 'piwik', 'http://apollo.lv/portal/search/?cof=FORID%3A11&q=piwik&search_where=www'), - array('http://bing.com/images/search', 'piwik', 'http://bing.com/images/search/?q=piwik'), - array('http://google.com', 'piwik', 'http://google.com/search?q=piwik'), - ); - } - - /** - * get search engine url from name and keyword - * - * @group Plugins - * - * @dataProvider getSearchEngineUrlFromUrlAndKeywordTestData - */ - public function testGetSearchEngineUrlFromUrlAndKeyword($url, $keyword, $expected) - { - include PIWIK_PATH_TEST_TO_ROOT . '/core/DataFiles/SearchEngines.php'; - $this->assertEquals($expected, \Piwik\Plugins\Referrers\getSearchEngineUrlFromUrlAndKeyword($url, $keyword)); - } - - /** - * Dataprovider for getSocialNetworkFromDomainTestData - */ - public function getSocialNetworkFromDomainTestData() - { - return array( - array('http://www.facebook.com', 'Facebook'), - array('http://www.facebook.com/piwik', 'Facebook'), - array('http://m.facebook.com', 'Facebook'), - array('https://m.facebook.com', 'Facebook'), - array('m.facebook.com', 'Facebook'), - array('http://lastfm.com.tr', 'Last.fm'), - array('http://t.co/test', 'Twitter'), - array('http://xxt.co/test', \Piwik\Piwik::translate('General_Unknown')), - array('asdfasdfadsf.com', \Piwik\Piwik::translate('General_Unknown')), - array('http://xwayn.com', \Piwik\Piwik::translate('General_Unknown')), - array('http://live.com/test', \Piwik\Piwik::translate('General_Unknown')), - ); - } - - /** - * @group Plugins - * - * @dataProvider getSocialNetworkFromDomainTestData - */ - public function testGetSocialNetworkFromDomain($url, $expected) - { - include PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php'; - $this->assertEquals($expected, \Piwik\Plugins\Referrers\getSocialNetworkFromDomain($url)); - } - - public function getSocialsLogoFromUrlTestData() - { - return array( - array('http://www.facebook.com', 'facebook.com.png'), - array('www.facebook.com', 'facebook.com.png',), - array('http://lastfm.com.tr', 'last.fm.png'), - array('http://asdfasdf.org/test', 'xx.png'), - array('http://www.google.com', 'xx.png'), - ); - } - - /** - * @group Plugins - * - * @dataProvider getSocialsLogoFromUrlTestData - */ - public function testGetSocialsLogoFromUrl($url, $expected) - { - include PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php'; - $this->assertContains($expected, \Piwik\Plugins\Referrers\getSocialsLogoFromUrl($url)); - } - - public function isSocialUrlTestData() - { - return array( - array('http://www.facebook.com', 'Facebook', true), - array('http://www.facebook.com', 'Twitter', false), - array('http://m.facebook.com', false, true), - array('http://lastfm.com.tr', 'Last.fm', true), - array('http://asdfasdf.org/test', false, false), - array('http://asdfasdf.com/test', 'Facebook', false), - ); - } - - /** - * @group Plugins - * - * @dataProvider isSocialUrlTestData - */ - public function testIsSocialUrl($url, $assumedSocial, $expected) - { - include PIWIK_INCLUDE_PATH . '/core/DataFiles/Socials.php'; - $this->assertEquals($expected, \Piwik\Plugins\Referrers\isSocialUrl($url, $assumedSocial)); - } - public function removeUrlProtocolTestData() { return array( diff --git a/plugins/Referrers/tests/Unit/SearchEngineTest.php b/plugins/Referrers/tests/Unit/SearchEngineTest.php new file mode 100644 index 0000000000..e9f0c926ea --- /dev/null +++ b/plugins/Referrers/tests/Unit/SearchEngineTest.php @@ -0,0 +1,173 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ + +namespace Piwik\Plugins\Referrers\tests; + +use Piwik\Plugins\Referrers\SearchEngine; +use Spyc; + +/** + * @group SearchEngine + */ +class SearchEngineTest extends \PHPUnit_Framework_TestCase +{ + public static function setUpBeforeClass() + { + // inject definitions to avoid database usage + $yml = file_get_contents(PIWIK_PATH_TEST_TO_ROOT . SearchEngine::DEFINITION_FILE); + SearchEngine::getInstance()->loadYmlData($yml); + parent::setUpBeforeClass(); + } + + public function getSearchEngineUrls() + { + return Spyc::YAMLLoad(PIWIK_PATH_TEST_TO_ROOT .'/tests/resources/extractSearchEngineInformationFromUrlTests.yml'); + } + + /** + * @dataProvider getSearchEngineUrls + */ + public function testExtractInformationFromUrl($url, $engine, $keywords) + { + $returnedValue = SearchEngine::getInstance()->extractInformationFromUrl($url); + + $expectedValue = false; + + if (!empty($engine)) { + $expectedValue = array('name' => $engine, 'keywords' => $keywords); + } + + $this->assertEquals($expectedValue, $returnedValue); + } + + public function testSearchEnginesDefinedCorrectly() + { + $searchEngines = array(); + foreach (SearchEngine::getInstance()->getDefinitions() as $host => $info) { + if (isset($info['backlink']) && $info['backlink'] !== false) { + $this->assertTrue(strrpos($info['backlink'], "{k}") !== false, $host . " search URL is not defined correctly, must contain the macro {k}"); + } + + if (!array_key_exists($info['name'], $searchEngines)) { + $searchEngines[$info['name']] = true; + + $this->assertTrue(strpos($host, '{}') === false, $host . " search URL is the master record and should not contain {}"); + } + + if (isset($info['charsets']) && $info['charsets'] !== false) { + $this->assertTrue(is_array($info['charsets']) || is_string($info['charsets']), $host . ' charsets must be either a string or an array'); + + if (is_string($info['charsets'])) { + $this->assertTrue(trim($info['charsets']) !== '', $host . ' charsets cannot be an empty string'); + $this->assertTrue(strpos($info['charsets'], ' ') === false, $host . ' charsets cannot contain spaces'); + + } + + if (is_array($info['charsets'])) { + $this->assertTrue(count($info['charsets']) > 0, $host . ' charsets cannot be an empty array'); + $this->assertTrue(strpos(serialize($info['charsets']), '""') === false, $host . ' charsets in array cannot be empty stringss'); + $this->assertTrue(strpos(serialize($info['charsets']), ' ') === false, $host . ' charsets in array cannot contain spaces'); + } + } + } + } + + /** + * Dataprovider for testGetBackLinkFromUrlAndKeyword + */ + public function getBackLinkFromUrlAndKeywordTestData() + { + return array( + array('http://apollo.lv/portal/search/', 'piwik', 'http://apollo.lv/portal/search/?cof=FORID%3A11&q=piwik&search_where=www'), + array('http://bing.com/images/search', 'piwik', 'http://bing.com/images/search/?q=piwik'), + array('http://google.com', 'piwik', 'http://google.com/search?q=piwik'), + ); + } + + /** + * get search engine url from name and keyword + * + * @dataProvider getBackLinkFromUrlAndKeywordTestData + */ + public function testGetBackLinkFromUrlAndKeyword($url, $keyword, $expected) + { + $this->assertEquals($expected, SearchEngine::getInstance()->getBackLinkFromUrlAndKeyword($url, $keyword)); + } + + /** + * Dataprovider serving all search engine data + */ + public function getAllSearchEngines() + { + $yml = file_get_contents(PIWIK_PATH_TEST_TO_ROOT . SearchEngine::DEFINITION_FILE); + SearchEngine::getInstance()->loadYmlData($yml); + $searchEngines = array(); + foreach (SearchEngine::getInstance()->getDefinitions() as $url => $searchEngine) { + $searchEngines[] = array($url, $searchEngine); + } + return $searchEngines; + } + + /** + * search engine has at least one keyword + * + * @dataProvider getAllSearchEngines + */ + public function testMissingSearchEngineKeyword($url, $searchEngine) + { + $name = parse_url('http://' . $url); + $this->assertTrue(!empty($searchEngine['params']), $name['host']); + } + + /** + * search engine is defined but there's no favicon + * + * @dataProvider getAllSearchEngines + */ + public function testMissingSearchEngineIcons($url, $searchEngine) + { + // Get list of existing favicons + $favicons = scandir(PIWIK_PATH_TEST_TO_ROOT . '/plugins/Referrers/images/searchEngines/'); + + // Get list of search engines and first appearing URL + static $searchEngines = array(); + + $name = parse_url('http://' . $url); + if (!array_key_exists($searchEngine['name'], $searchEngines)) { + $searchEngines[$searchEngine['name']] = $url; + + $this->assertTrue(in_array($name['host'] . '.png', $favicons), $name['host']); + } + } + + /** + * favicon exists but there's no corresponding search engine defined + */ + public function testObsoleteSearchEngineIcons() + { + // Get list of search engines and first appearing URL + $searchEngines = array(); + foreach (SearchEngine::getInstance()->getDefinitions() as $url => $searchEngine) { + $name = parse_url('http://' . $url); + if (!array_key_exists($name['host'], $searchEngines)) { + $searchEngines[$name['host']] = true; + } + } + + // Get list of existing favicons + $favicons = scandir(PIWIK_PATH_TEST_TO_ROOT . '/plugins/Referrers/images/searchEngines/'); + foreach ($favicons as $name) { + if ($name[0] == '.' || strpos($name, 'xx.') === 0) { + continue; + } + + $host = substr($name, 0, -4); + $this->assertTrue(array_key_exists($host, $searchEngines), $host); + } + } +}
\ No newline at end of file diff --git a/plugins/Referrers/tests/Unit/SocialTest.php b/plugins/Referrers/tests/Unit/SocialTest.php new file mode 100644 index 0000000000..8e8f35e0f9 --- /dev/null +++ b/plugins/Referrers/tests/Unit/SocialTest.php @@ -0,0 +1,97 @@ +<?php +/** + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ + +namespace Piwik\Plugins\Referrers\tests; + +use Piwik\Plugins\Referrers\Social; +use Spyc; + +/** + * @group Social + * @group Plugins + */ +class SocialTest extends \PHPUnit_Framework_TestCase +{ + public static function setUpBeforeClass() + { + // inject definitions to avoid database usage + $yml = file_get_contents(PIWIK_PATH_TEST_TO_ROOT . Social::DEFINITION_FILE); + Social::getInstance()->loadYmlData($yml); + parent::setUpBeforeClass(); + } + + public function isSocialUrlTestData() + { + return array( + array('http://www.facebook.com', 'Facebook', true), + array('http://www.facebook.com', 'Twitter', false), + array('http://m.facebook.com', false, true), + array('http://lastfm.com.tr', 'Last.fm', true), + array('http://asdfasdf.org/test', false, false), + array('http://asdfasdf.com/test', 'Facebook', false), + ); + } + + /** + * @dataProvider isSocialUrlTestData + */ + public function testIsSocialUrl($url, $assumedSocial, $expected) + { + $this->assertEquals($expected, Social::getInstance()->isSocialUrl($url, $assumedSocial)); + } + + + /** + * Dataprovider for getSocialNetworkFromDomainTestData + */ + public function getSocialNetworkFromDomainTestData() + { + return array( + array('http://www.facebook.com', 'Facebook'), + array('http://www.facebook.com/piwik', 'Facebook'), + array('http://m.facebook.com', 'Facebook'), + array('https://m.facebook.com', 'Facebook'), + array('m.facebook.com', 'Facebook'), + array('http://lastfm.com.tr', 'Last.fm'), + array('http://t.co/test', 'Twitter'), + array('http://xxt.co/test', \Piwik\Piwik::translate('General_Unknown')), + array('asdfasdfadsf.com', \Piwik\Piwik::translate('General_Unknown')), + array('http://xwayn.com', \Piwik\Piwik::translate('General_Unknown')), + array('http://live.com/test', \Piwik\Piwik::translate('General_Unknown')), + ); + } + + /** + * @dataProvider getSocialNetworkFromDomainTestData + */ + public function testGetSocialNetworkFromDomain($url, $expected) + { + $this->assertEquals($expected, Social::getInstance()->getSocialNetworkFromDomain($url)); + } + + public function getLogoFromUrlTestData() + { + return array( + array('http://www.facebook.com', 'facebook.com.png'), + array('www.facebook.com', 'facebook.com.png',), + array('http://lastfm.com.tr', 'last.fm.png'), + array('http://asdfasdf.org/test', 'xx.png'), + array('http://www.google.com', 'xx.png'), + ); + } + + /** + * @group Plugins + * + * @dataProvider getLogoFromUrlTestData + */ + public function testGetLogoFromUrl($url, $expected) + { + $this->assertContains($expected, Social::getInstance()->getLogoFromUrl($url)); + } +}
\ No newline at end of file diff --git a/plugins/SEO/Metric/Alexa.php b/plugins/SEO/Metric/Alexa.php index 87c5785d36..e8f7956241 100644 --- a/plugins/SEO/Metric/Alexa.php +++ b/plugins/SEO/Metric/Alexa.php @@ -10,6 +10,7 @@ namespace Piwik\Plugins\SEO\Metric; use Piwik\Http; use Piwik\NumberFormatter; +use Piwik\Plugins\Referrers\SearchEngine; use Psr\Log\LoggerInterface; /** @@ -42,7 +43,7 @@ class Alexa implements MetricsProvider $value = null; } - $logo = \Piwik\Plugins\Referrers\getSearchEngineLogoFromUrl('http://alexa.com'); + $logo = SearchEngine::getInstance()->getLogoFromUrl('http://alexa.com'); $link = self::LINK . urlencode($domain); return array( diff --git a/plugins/SEO/Metric/Bing.php b/plugins/SEO/Metric/Bing.php index e85d585c13..71553ffc6e 100644 --- a/plugins/SEO/Metric/Bing.php +++ b/plugins/SEO/Metric/Bing.php @@ -10,6 +10,7 @@ namespace Piwik\Plugins\SEO\Metric; use Piwik\Http; use Piwik\NumberFormatter; +use Piwik\Plugins\Referrers\SearchEngine; use Psr\Log\LoggerInterface; /** @@ -46,7 +47,7 @@ class Bing implements MetricsProvider $pageCount = null; } - $logo = \Piwik\Plugins\Referrers\getSearchEngineLogoFromUrl('http://bing.com'); + $logo = SearchEngine::getInstance()->getLogoFromUrl('http://bing.com'); return array( new Metric('bing-index', 'SEO_Bing_IndexedPages', $pageCount, $logo, null, null, 'General_Pages') diff --git a/plugins/SEO/Metric/Dmoz.php b/plugins/SEO/Metric/Dmoz.php index d21be1e5ca..c7b8860a89 100644 --- a/plugins/SEO/Metric/Dmoz.php +++ b/plugins/SEO/Metric/Dmoz.php @@ -10,6 +10,7 @@ namespace Piwik\Plugins\SEO\Metric; use Piwik\Http; use Piwik\NumberFormatter; +use Piwik\Plugins\Referrers\SearchEngine; use Psr\Log\LoggerInterface; /** @@ -53,7 +54,7 @@ class Dmoz implements MetricsProvider $value = null; } - $logo = \Piwik\Plugins\Referrers\getSearchEngineLogoFromUrl('http://dmoz.org'); + $logo = SearchEngine::getInstance()->getLogoFromUrl('http://dmoz.org'); return array( new Metric('dmoz', 'SEO_Dmoz', $value, $logo) diff --git a/plugins/SEO/Metric/Google.php b/plugins/SEO/Metric/Google.php index cec1d96af9..7abd82b371 100644 --- a/plugins/SEO/Metric/Google.php +++ b/plugins/SEO/Metric/Google.php @@ -10,6 +10,7 @@ namespace Piwik\Plugins\SEO\Metric; use Piwik\Http; use Piwik\NumberFormatter; +use Piwik\Plugins\Referrers\SearchEngine; use Psr\Log\LoggerInterface; /** @@ -37,7 +38,7 @@ class Google implements MetricsProvider $pageCount = $this->fetchIndexedPagesCount($domain); $pageRank = $this->fetchPageRank($domain); - $logo = \Piwik\Plugins\Referrers\getSearchEngineLogoFromUrl('http://google.com'); + $logo = SearchEngine::getInstance()->getLogoFromUrl('http://google.com'); return array( new Metric('google-index', 'SEO_Google_IndexedPages', $pageCount, $logo, null, null, 'General_Pages'), diff --git a/tests/PHPUnit/Framework/Fixture.php b/tests/PHPUnit/Framework/Fixture.php index 04c52371da..29b4625465 100644 --- a/tests/PHPUnit/Framework/Fixture.php +++ b/tests/PHPUnit/Framework/Fixture.php @@ -243,8 +243,6 @@ class Fixture extends \PHPUnit_Framework_Assert static::fail("TEST INITIALIZATION FAILED: " . $e->getMessage() . "\n" . $e->getTraceAsString()); } - include "DataFiles/SearchEngines.php"; - include "DataFiles/Socials.php"; include "DataFiles/Providers.php"; if (!$this->isFixtureSetUp()) { diff --git a/tests/PHPUnit/Unit/CommonTest.php b/tests/PHPUnit/Unit/CommonTest.php index 8aa85ed550..d8817ec3db 100644 --- a/tests/PHPUnit/Unit/CommonTest.php +++ b/tests/PHPUnit/Unit/CommonTest.php @@ -465,39 +465,4 @@ class CommonTest extends PHPUnit_Framework_TestCase { $this->assertEquals($expected, Common::extractLanguageCodeFromBrowserLanguage($browserLanguage, $validLanguages), "test with {$browserLanguage} failed, expected {$expected}"); } - - public function testSearchEnginesDefinedCorrectly() - { - include "DataFiles/SearchEngines.php"; - - $searchEngines = array(); - foreach ($GLOBALS['Piwik_SearchEngines'] as $host => $info) { - if (isset($info[2]) && $info[2] !== false) { - $this->assertTrue(strrpos($info[2], "{k}") !== false, $host . " search URL is not defined correctly, must contain the macro {k}"); - } - - if (!array_key_exists($info[0], $searchEngines)) { - $searchEngines[$info[0]] = true; - - $this->assertTrue(strpos($host, '{}') === false, $host . " search URL is the master record and should not contain {}"); - } - - if (isset($info[3]) && $info[3] !== false) { - $this->assertTrue(is_array($info[3]) || is_string($info[3]), $host . ' encoding must be either a string or an array'); - - if (is_string($info[3])) { - $this->assertTrue(trim($info[3]) !== '', $host . ' encoding cannot be an empty string'); - $this->assertTrue(strpos($info[3], ' ') === false, $host . ' encoding cannot contain spaces'); - - } - - if (is_array($info[3])) { - $this->assertTrue(count($info[3]) > 0, $host . ' encodings cannot be an empty array'); - $this->assertTrue(strpos(serialize($info[3]), '""') === false, $host . ' encodings in array cannot be empty stringss'); - $this->assertTrue(strpos(serialize($info[3]), ' ') === false, $host . ' encodings in array cannot contain spaces'); - } - } - } - } - } diff --git a/tests/PHPUnit/Unit/Metrics/FormatterTest.php b/tests/PHPUnit/Unit/Metrics/FormatterTest.php index 0bcef84c5f..d9ccc18c6d 100644 --- a/tests/PHPUnit/Unit/Metrics/FormatterTest.php +++ b/tests/PHPUnit/Unit/Metrics/FormatterTest.php @@ -190,9 +190,9 @@ class FormatterTest extends \PHPUnit_Framework_TestCase array(100, array('1 min 40s', '00:01:40')), array(3600, array('1 hours 0 min', '01:00:00')), array(3700, array('1 hours 1 min', '01:01:40')), - array(86400 + 3600 * 10, array('1 days 10 hours', '34:00:00')), - array(86400 * 365, array('365 days 0 hours', '8760:00:00')), - array((86400 * (365.25 + 10)), array('1 years 10 days', '9006:00:00')), + array(86400 + 3600 * 10, array('1 days 10 hours', '1 days 10:00:00')), + array(86400 * 365, array('365 days 0 hours', '365 days 00:00:00')), + array((86400 * (365.25 + 10)), array('1 years 10 days', '375 days 06:00:00')), array(1.342, array('1.34s', '00:00:01.34')), array(.342, array('0.34s', '00:00:00.34')), array(.02, array('0.02s', '00:00:00.02')), @@ -202,7 +202,7 @@ class FormatterTest extends \PHPUnit_Framework_TestCase array(1.2, array('1.2s', '00:00:01.20')), array(122.1, array('2 min 2.1s', '00:02:02.10')), array(-122.1, array('-2 min 2.1s', '-00:02:02.10')), - array(86400 * -365, array('-365 days 0 hours', '-8760:00:00')) + array(86400 * -365, array('-365 days 0 hours', '-365 days 00:00:00')) ); } @@ -222,4 +222,4 @@ class FormatterTest extends \PHPUnit_Framework_TestCase SitesManagerAPI::setSingletonInstance($mock); } -}
\ No newline at end of file +} diff --git a/tests/PHPUnit/Unit/UrlHelperTest.php b/tests/PHPUnit/Unit/UrlHelperTest.php index e972d20d54..cc15d6eeb1 100644 --- a/tests/PHPUnit/Unit/UrlHelperTest.php +++ b/tests/PHPUnit/Unit/UrlHelperTest.php @@ -150,32 +150,6 @@ class UrlHelperTest extends \PHPUnit_Framework_TestCase } /** - * Dataprovider for testExtractSearchEngineInformationFromUrl - */ - public function getSearchEngineUrls() - { - return Spyc::YAMLLoad(PIWIK_PATH_TEST_TO_ROOT .'/tests/resources/extractSearchEngineInformationFromUrlTests.yml'); - } - - /** - * @dataProvider getSearchEngineUrls - * @group Core - */ - public function testExtractSearchEngineInformationFromUrl($url, $engine, $keywords) - { - $this->includeDataFilesForSearchEngineTest(); - $returnedValue = UrlHelper::extractSearchEngineInformationFromUrl($url); - - $exptectedValue = false; - - if (!empty($engine)) { - $exptectedValue = array('name' => $engine, 'keywords' => $keywords); - } - - $this->assertEquals($exptectedValue, $returnedValue); - } - - /** * Dataprovider for testGetLossyUrl */ public function getLossyUrls() @@ -203,11 +177,6 @@ class UrlHelperTest extends \PHPUnit_Framework_TestCase $this->assertEquals($expected, UrlHelper::getLossyUrl($input)); } - private function includeDataFilesForSearchEngineTest() - { - include "DataFiles/SearchEngines.php"; - } - /** * @group Core */ |