From fb91155043c2dd269f32e27962e3cbdb00ad0aad Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Sun, 21 Dec 2014 23:13:16 +0100 Subject: When importing visits on a day different from the visits day, invalidate the archived reports --- core/CronArchive.php | 44 +- core/DataAccess/ArchiveInvalidator.php | 78 +- core/Site.php | 7 +- core/Tracker/Cache.php | 2 +- core/Tracker/Visit.php | 24 + misc/log-analytics/import_logs.py | 24 - plugins/CoreAdminHome/tests/Framework/Mock/API.php | 26 + plugins/SitesManager/API.php | 4 +- plugins/SitesManager/SitesManager.php | 28 +- plugins/SitesManager/tests/Integration/ApiTest.php | 1078 ++++++++++++++++++++ .../tests/Integration/SitesManagerTest.php | 985 +----------------- .../PHPUnit/Framework/TestCase/SystemTestCase.php | 3 + tests/PHPUnit/Integration/CronArchiveTest.php | 69 ++ .../DataAccess/ArchiveInvalidatorTest.php | 181 ++++ tests/PHPUnit/Integration/Tracker/VisitTest.php | 100 +- 15 files changed, 1659 insertions(+), 994 deletions(-) create mode 100644 plugins/CoreAdminHome/tests/Framework/Mock/API.php create mode 100644 plugins/SitesManager/tests/Integration/ApiTest.php create mode 100644 tests/PHPUnit/Integration/CronArchiveTest.php create mode 100644 tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php diff --git a/core/CronArchive.php b/core/CronArchive.php index 3348966ab0..743b88b888 100644 --- a/core/CronArchive.php +++ b/core/CronArchive.php @@ -12,10 +12,12 @@ use Exception; use Piwik\ArchiveProcessor\Rules; use Piwik\CronArchive\FixedSiteIds; use Piwik\CronArchive\SharedSiteIds; +use Piwik\DataAccess\ArchiveInvalidator; use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Metrics\Formatter; use Piwik\Period\Factory as PeriodFactory; use Piwik\DataAccess\InvalidatedReports; +use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeAPI; use Piwik\Plugins\SitesManager\API as APISitesManager; use Piwik\Plugins\UsersManager\API as APIUsersManager; use Piwik\Plugins\UsersManager\UserPreferences; @@ -85,6 +87,8 @@ class CronArchive private $lastSuccessRunTimestamp = false; private $errors = array(); + private $apiToInvalidateArchivedReport; + const NO_ERROR = "no error"; public $testmode = false; @@ -256,6 +260,8 @@ class CronArchive $this->log("- Will only process the following periods: " . implode(", ", $this->shouldArchiveOnlySpecificPeriods) . " (--force-periods)"); } + $this->invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain(); + $websitesIds = $this->initWebsiteIds(); $this->filterWebsiteIds($websitesIds); @@ -951,6 +957,40 @@ class CronArchive Piwik::postEvent('CronArchive.filterWebsiteIds', array(&$websiteIds)); } + /** + * @internal + */ + public function setApiToInvalidateArchivedReport($api) + { + $this->apiToInvalidateArchivedReport = $api; + } + + private function getApiToInvalidateArchivedReport() + { + if ($this->apiToInvalidateArchivedReport) { + return $this->apiToInvalidateArchivedReport; + } + + return CoreAdminHomeAPI::getInstance(); + } + + public function invalidateArchivedReportsForSitesThatNeedToBeArchivedAgain() + { + $invalidator = new ArchiveInvalidator(); + $sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + foreach ($sitesPerDays as $date => $siteIds) { + $listSiteIds = implode(',', $siteIds); + + try { + $this->log('Will invalidate archived reports for ' . $date . ' for following siteIds: ' . $listSiteIds); + $this->getApiToInvalidateArchivedReport()->invalidateArchivedReports($siteIds, $date); + } catch (Exception $e) { + $this->log('Failed to invalidate archived reports: ' . $e->getMessage()); + } + } + } + /** * Returns the list of sites to loop over and archive. * @return array @@ -998,7 +1038,7 @@ class CronArchive return in_array($token_auth, $this->validTokenAuths); } - private function initPiwikHost($piwikUrl = false) + protected function initPiwikHost($piwikUrl = false) { // If core:archive command run as a web cron, we use the current hostname+path if (empty($piwikUrl)) { @@ -1150,7 +1190,7 @@ class CronArchive /** * Test that the specified piwik URL is a valid Piwik endpoint. */ - private function checkPiwikUrlIsValid() + protected function checkPiwikUrlIsValid() { $response = $this->request("?module=API&method=API.getDefaultMetricTranslations&format=original&serialize=1"); $responseUnserialized = @unserialize($response); diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/DataAccess/ArchiveInvalidator.php index 2d1a87daf1..962f9d4386 100644 --- a/core/DataAccess/ArchiveInvalidator.php +++ b/core/DataAccess/ArchiveInvalidator.php @@ -9,9 +9,9 @@ namespace Piwik\DataAccess; - use Piwik\Date; use Piwik\Db; +use Piwik\Option; use Piwik\Plugins\PrivacyManager\PrivacyManager; use Piwik\Period; use Piwik\Period\Week; @@ -32,6 +32,73 @@ class ArchiveInvalidator { private $minimumDateWithLogs = false; private $invalidDates = array(); + private $rememberArchivedReportIdStart = 'report_to_invalidate_'; + + public function rememberToInvalidateArchivedReportsLater($idSite, Date $date) + { + $key = $this->buildRememberArchivedReportId($idSite, $date->toString()); + $value = Option::get($key); + + // we do not really have to get the value first. we could simply always try to call set() and it would update or + // insert the record if needed but we do not want to lock the table (especially since there are still some + // MyISAM installations) + + if (false === $value) { + Option::set($key, '1'); + } + } + + public function getRememberedArchivedReportsThatShouldBeInvalidated() + { + $reports = Option::getLike($this->rememberArchivedReportIdStart . '%_%'); + + $sitesPerDay = array(); + + foreach ($reports as $report => $value) { + $report = str_replace($this->rememberArchivedReportIdStart, '', $report); + $report = explode('_', $report); + $siteId = (int) $report[0]; + $date = $report[1]; + + if (empty($sitesPerDay[$date])) { + $sitesPerDay[$date] = array(); + } + + $sitesPerDay[$date][] = $siteId; + } + + return $sitesPerDay; + } + + private function buildRememberArchivedReportId($idSite, $date) + { + $id = $this->buildRememberArchivedReportIdForSite($idSite); + $id .= '_' . trim($date); + + return $id; + } + + private function buildRememberArchivedReportIdForSite($idSite) + { + return $this->rememberArchivedReportIdStart . (int) $idSite; + } + + public function forgetRememberedArchivedReportsToInvalidateForSite($idSite) + { + $id = $this->buildRememberArchivedReportIdForSite($idSite) . '_%'; + Option::deleteLike($id); + } + + /** + * @internal + */ + public function forgetRememberedArchivedReportsToInvalidate($idSite, Date $date) + { + $id = $this->buildRememberArchivedReportId($idSite, $date->toString()); + + Option::delete($id); + } + /** * @param $idSites array * @param $dates string @@ -52,6 +119,12 @@ class ArchiveInvalidator { $this->persistInvalidatedArchives($idSites, $datesByMonth); + foreach ($idSites as $idSite) { + foreach ($datesToInvalidate as $date) { + $this->forgetRememberedArchivedReportsToInvalidate($idSite, $date); + } + } + return $this->makeOutputLogs(); } @@ -105,7 +178,7 @@ class ArchiveInvalidator { /** * Ensure the specified dates are valid. * Store invalid date so we can log them - * @param $dates string + * @param array $dates * @return Date[] */ private function getDatesToInvalidateFromString($dates) @@ -129,6 +202,7 @@ class ArchiveInvalidator { $this->invalidDates[] = $theDate; } } + return $toInvalidate; } diff --git a/core/Site.php b/core/Site.php index 4818590f82..6ac0c33f57 100644 --- a/core/Site.php +++ b/core/Site.php @@ -133,7 +133,12 @@ class Site public static function setSitesFromArray($sites) { foreach ($sites as $site) { - self::setSite($site['idsite'], $site); + $idSite = null; + if (!empty($site['idsite'])) { + $idSite = $site['idsite']; + } + + self::setSite($idSite, $site); } } diff --git a/core/Tracker/Cache.php b/core/Tracker/Cache.php index 3fa360986e..22b57a9cc2 100644 --- a/core/Tracker/Cache.php +++ b/core/Tracker/Cache.php @@ -60,7 +60,7 @@ class Cache return array(); } - $idSite = (int)$idSite; + $idSite = (int) $idSite; if ($idSite <= 0) { return array(); } diff --git a/core/Tracker/Visit.php b/core/Tracker/Visit.php index 41db09e557..f6ae0364c8 100644 --- a/core/Tracker/Visit.php +++ b/core/Tracker/Visit.php @@ -11,6 +11,9 @@ namespace Piwik\Tracker; use Piwik\Common; use Piwik\Config; +use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Date; +use Piwik\Exception\UnexpectedWebsiteFoundException; use Piwik\Network\IPUtils; use Piwik\Piwik; use Piwik\Plugin\Dimension\VisitDimension; @@ -218,6 +221,8 @@ class Visit implements VisitInterface } unset($this->goalManager); unset($action); + + $this->markArchivedReportsAsInvalidIfArchiveAlreadyFinished(); } /** @@ -626,4 +631,23 @@ class Visit implements VisitInterface return !$visitor->isVisitorKnown(); } + + private function markArchivedReportsAsInvalidIfArchiveAlreadyFinished() + { + $idSite = (int) $this->request->getIdSite(); + $time = $this->request->getCurrentTimestamp(); + + try { + $site = Cache::getCacheWebsiteAttributes($idSite); + } catch (UnexpectedWebsiteFoundException $e) { + return; + } + + $date = Date::factory((int) $time, $site['timezone']); + + if (!$date->isToday()) { // we don't have to handle in case date is in future as it is not allowed by tracker + $invalidReport = new ArchiveInvalidator(); + $invalidReport->rememberToInvalidateArchivedReportsLater($idSite, $date); + } + } } diff --git a/misc/log-analytics/import_logs.py b/misc/log-analytics/import_logs.py index 315b2ac1a1..3714420363 100755 --- a/misc/log-analytics/import_logs.py +++ b/misc/log-analytics/import_logs.py @@ -1443,26 +1443,6 @@ class Recorder(object): return response['message'] - @staticmethod - def invalidate_reports(): - if config.options.dry_run or not stats.dates_recorded: - return - - if config.options.invalidate_dates is not None: - dates = [date for date in config.options.invalidate_dates.split(',') if date] - else: - dates = [date.strftime('%Y-%m-%d') for date in stats.dates_recorded] - if dates: - print '\nPurging Piwik archives for dates: ' + ' '.join(dates) - result = piwik.call_api( - 'CoreAdminHome.invalidateArchivedReports', - dates=','.join(dates), - idSites=','.join(str(site_id) for site_id in stats.piwik_sites), - ) - print('\nTo re-process these reports with your newly imported data, execute the following command: \n' - '$ /path/to/piwik/console core:archive --url=http://example/piwik --force-all-websites --force-all-periods=315576000 --force-date-last-n=1000' - '\nReference: http://piwik.org/docs/setup-auto-archiving/ ') - class Hit(object): """ It's a simple container. @@ -1899,10 +1879,6 @@ def main(): if config.options.show_progress: stats.stop_monitor() - try: - Recorder.invalidate_reports() - except Piwik.Error, e: - pass stats.print_summary() def fatal_error(error, filename=None, lineno=None): diff --git a/plugins/CoreAdminHome/tests/Framework/Mock/API.php b/plugins/CoreAdminHome/tests/Framework/Mock/API.php new file mode 100644 index 0000000000..70070c50e9 --- /dev/null +++ b/plugins/CoreAdminHome/tests/Framework/Mock/API.php @@ -0,0 +1,26 @@ +invalidatedReports[] = func_get_args(); + } + + public function getInvalidatedReports() + { + return $this->invalidatedReports; + } +} diff --git a/plugins/SitesManager/API.php b/plugins/SitesManager/API.php index 1e7ffb2867..c39c6fb52f 100644 --- a/plugins/SitesManager/API.php +++ b/plugins/SitesManager/API.php @@ -182,6 +182,7 @@ class API extends \Piwik\Plugin\API $site = $this->getModel()->getSiteFromId($idSite); Site::setSitesFromArray(array($site)); + return $site; } @@ -599,9 +600,6 @@ class API extends \Piwik\Plugin\API $this->getModel()->deleteSite($idSite); - // we do not delete logs here on purpose (you can run these queries on the log_ tables to delete all data) - Cache::deleteCacheWebsiteAttributes($idSite); - /** * Triggered after a site has been deleted. * diff --git a/plugins/SitesManager/SitesManager.php b/plugins/SitesManager/SitesManager.php index a7ee82c6fc..f06ff69948 100644 --- a/plugins/SitesManager/SitesManager.php +++ b/plugins/SitesManager/SitesManager.php @@ -7,6 +7,8 @@ * */ namespace Piwik\Plugins\SitesManager; +use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Tracker\Cache; /** * @@ -27,9 +29,19 @@ class SitesManager extends \Piwik\Plugin 'AssetManager.getStylesheetFiles' => 'getStylesheetFiles', 'Tracker.Cache.getSiteAttributes' => 'recordWebsiteDataInCache', 'Translate.getClientSideTranslationKeys' => 'getClientSideTranslationKeys', + 'SitesManager.deleteSite.end' => 'onSiteDeleted' ); } + public function onSiteDeleted($idSite) + { + // we do not delete logs here on purpose (you can run these queries on the log_ tables to delete all data) + Cache::deleteCacheWebsiteAttributes($idSite); + + $archiveInvalidator = new ArchiveInvalidator(); + $archiveInvalidator->forgetRememberedArchivedReportsToInvalidateForSite($idSite); + } + /** * Get CSS files */ @@ -64,7 +76,7 @@ class SitesManager extends \Piwik\Plugin */ public function recordWebsiteDataInCache(&$array, $idSite) { - $idSite = (int)$idSite; + $idSite = (int) $idSite; // add the 'hosts' entry in the website array $array['hosts'] = $this->getTrackerHosts($idSite); @@ -77,6 +89,20 @@ class SitesManager extends \Piwik\Plugin $array['sitesearch'] = $website['sitesearch']; $array['sitesearch_keyword_parameters'] = $this->getTrackerSearchKeywordParameters($website); $array['sitesearch_category_parameters'] = $this->getTrackerSearchCategoryParameters($website); + $array['timezone'] = $this->getTimezoneFromWebsite($website); + } + + /** + * Returns whether we should keep URL fragments for a specific site. + * + * @param array $site DB data for the site. + * @return bool + */ + private static function getTimezoneFromWebsite($site) + { + if (!empty($site['timezone'])) { + return $site['timezone']; + } } /** diff --git a/plugins/SitesManager/tests/Integration/ApiTest.php b/plugins/SitesManager/tests/Integration/ApiTest.php new file mode 100644 index 0000000000..d4a4b6c87b --- /dev/null +++ b/plugins/SitesManager/tests/Integration/ApiTest.php @@ -0,0 +1,1078 @@ + exception + * + * @group Plugins + */ + public function testAddSiteEmptyName() + { + try { + API::getInstance()->addSite("", array("http://piwik.net")); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * DataProvider for testAddSiteWrongUrls + */ + public function getInvalidUrlData() + { + return array( + array(array()), // no urls + array(array("")), + array(""), + array("httpww://piwik.net"), + array("httpww://piwik.net/gqg~#"), + ); + } + + /** + * wrong urls -> exception + * + * @dataProvider getInvalidUrlData + * @group Plugins + */ + public function testAddSiteWrongUrls($url) + { + try { + API::getInstance()->addSite("name", $url); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * Test with valid IPs + * + * @group Plugins + */ + public function testAddSiteExcludedIpsAndtimezoneAndCurrencyAndExcludedQueryParametersValid() + { + $ips = '1.2.3.4,1.1.1.*,1.2.*.*,1.*.*.*'; + $timezone = 'Europe/Paris'; + $currency = 'EUR'; + $excludedQueryParameters = 'p1,P2, P33333'; + $expectedExcludedQueryParameters = 'p1,P2,P33333'; + $excludedUserAgents = " p1,P2, \nP3333 "; + $expectedExcludedUserAgents = "p1,P2,P3333"; + $expectedWebsiteType = 'mobile-\'app'; + $keepUrlFragment = 1; + $idsite = API::getInstance()->addSite("name", "http://piwik.net/", $ecommerce = 1, + $siteSearch = 1, $searchKeywordParameters = 'search,param', $searchCategoryParameters = 'cat,category', + $ips, $excludedQueryParameters, $timezone, $currency, $group = null, $startDate = null, $excludedUserAgents, + $keepUrlFragment, $expectedWebsiteType); + $siteInfo = API::getInstance()->getSiteFromId($idsite); + $this->assertEquals($ips, $siteInfo['excluded_ips']); + $this->assertEquals($timezone, $siteInfo['timezone']); + $this->assertEquals($currency, $siteInfo['currency']); + $this->assertEquals($ecommerce, $siteInfo['ecommerce']); + $this->assertTrue(Site::isEcommerceEnabledFor($idsite)); + $this->assertEquals($siteSearch, $siteInfo['sitesearch']); + $this->assertTrue(Site::isSiteSearchEnabledFor($idsite)); + $this->assertEquals($expectedWebsiteType, $siteInfo['type']); + $this->assertEquals($expectedWebsiteType, Site::getTypeFor($idsite)); + + $this->assertEquals($searchKeywordParameters, $siteInfo['sitesearch_keyword_parameters']); + $this->assertEquals($searchCategoryParameters, $siteInfo['sitesearch_category_parameters']); + $this->assertEquals($expectedExcludedQueryParameters, $siteInfo['excluded_parameters']); + $this->assertEquals($expectedExcludedUserAgents, $siteInfo['excluded_user_agents']); + } + + /** + * dataProvider for testAddSiteExcludedIpsNotValid + */ + public function getInvalidIPsData() + { + return array( + array('35817587341'), + array('ieagieha'), + array('1.2.3'), + array('*.1.1.1'), + array('*.*.1.1'), + array('*.*.*.1'), + array('1.1.1.1.1'), + ); + } + + /** + * Test with invalid IPs + * + * @dataProvider getInvalidIPsData + * @group Plugins + */ + public function testAddSiteExcludedIpsNotValid($ip) + { + try { + API::getInstance()->addSite("name", "http://piwik.net/", $ecommerce = 0, + $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, $ip); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * one url -> one main_url and nothing inserted as alias urls + * + * @group Plugins + */ + public function testAddSiteOneUrl() + { + $url = "http://piwik.net/"; + $urlOK = "http://piwik.net"; + $idsite = API::getInstance()->addSite("name", $url); + $this->assertInternalType(PHPUnit_Framework_Constraint_IsType::TYPE_INT, $idsite); + + $siteInfo = API::getInstance()->getSiteFromId($idsite); + $this->assertEquals($urlOK, $siteInfo['main_url']); + $this->assertEquals(date('Y-m-d'), date('Y-m-d', strtotime($siteInfo['ts_created']))); + + $siteUrls = API::getInstance()->getSiteUrlsFromId($idsite); + $this->assertEquals(1, count($siteUrls)); + } + + /** + * several urls -> one main_url and others as alias urls + * + * @group Plugins + */ + public function testAddSiteSeveralUrls() + { + $urls = array("http://piwik.net/", "http://piwik.com", "https://piwik.net/test/"); + $urlsOK = array("http://piwik.net", "http://piwik.com", "https://piwik.net/test"); + $idsite = API::getInstance()->addSite("super website", $urls); + $this->assertInternalType(PHPUnit_Framework_Constraint_IsType::TYPE_INT, $idsite); + + $siteInfo = API::getInstance()->getSiteFromId($idsite); + $this->assertEquals($urlsOK[0], $siteInfo['main_url']); + + $siteUrls = API::getInstance()->getSiteUrlsFromId($idsite); + $this->assertEquals($urlsOK, $siteUrls); + } + + /** + * strange name + * + * @group Plugins + */ + public function testAddSiteStrangeName() + { + $name = "supertest(); ~@@()''!£\$'%%^'!£ போ"; + $idsite = API::getInstance()->addSite($name, "http://piwik.net"); + $this->assertInternalType(PHPUnit_Framework_Constraint_IsType::TYPE_INT, $idsite); + + $siteInfo = API::getInstance()->getSiteFromId($idsite); + $this->assertEquals($name, $siteInfo['name']); + + } + + /** + * adds a site + * use by several other unit tests + */ + protected function _addSite() + { + $name = "website "; + $idsite = API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")); + $this->assertInternalType(PHPUnit_Framework_Constraint_IsType::TYPE_INT, $idsite); + + $siteInfo = API::getInstance()->getSiteFromId($idsite); + $this->assertEquals($name, $siteInfo['name']); + $this->assertEquals("http://piwik.net", $siteInfo['main_url']); + + $siteUrls = API::getInstance()->getSiteUrlsFromId($idsite); + $this->assertEquals(array("http://piwik.net", "http://piwik.com/test"), $siteUrls); + + return $idsite; + } + + /** + * no duplicate -> all the urls are saved + * + * @group Plugins + */ + public function testAddSiteUrlsnoDuplicate() + { + $idsite = $this->_addSite(); + + $siteUrlsBefore = API::getInstance()->getSiteUrlsFromId($idsite); + + $toAdd = array("http://piwik1.net", + "http://piwik2.net", + "http://piwik3.net/test/", + "http://localhost/test", + "http://localho5.st/test", + "http://l42578gqege.f4", + "http://super.com/test/test/atqata675675/te" + ); + $toAddValid = array("http://piwik1.net", + "http://piwik2.net", + "http://piwik3.net/test", + "http://localhost/test", + "http://localho5.st/test", + "http://l42578gqege.f4", + "http://super.com/test/test/atqata675675/te"); + + $insertedUrls = API::getInstance()->addSiteAliasUrls($idsite, $toAdd); + $this->assertEquals(count($toAdd), $insertedUrls); + + $siteUrlsAfter = API::getInstance()->getSiteUrlsFromId($idsite); + + $shouldHave = array_merge($siteUrlsBefore, $toAddValid); + sort($shouldHave); + + sort($siteUrlsAfter); + + $this->assertEquals($shouldHave, $siteUrlsAfter); + } + + /** + * duplicate -> don't save the already existing URLs + * + * @group Plugins + */ + public function testAddSiteUrlsDuplicate() + { + $idsite = $this->_addSite(); + + $siteUrlsBefore = API::getInstance()->getSiteUrlsFromId($idsite); + + $toAdd = array_merge($siteUrlsBefore, array("http://piwik1.net", "http://piwik2.net")); + + $insertedUrls = API::getInstance()->addSiteAliasUrls($idsite, $toAdd); + $this->assertEquals(count($toAdd) - count($siteUrlsBefore), $insertedUrls); + + $siteUrlsAfter = API::getInstance()->getSiteUrlsFromId($idsite); + + $shouldHave = $toAdd; + sort($shouldHave); + + sort($siteUrlsAfter); + + $this->assertEquals($shouldHave, $siteUrlsAfter); + } + + /** + * case empty array => nothing happens + * + * @group Plugins + */ + public function testAddSiteUrlsNoUrlsToAdd1() + { + $idsite = $this->_addSite(); + + $siteUrlsBefore = API::getInstance()->getSiteUrlsFromId($idsite); + + $toAdd = array(); + + $insertedUrls = API::getInstance()->addSiteAliasUrls($idsite, $toAdd); + $this->assertEquals(count($toAdd), $insertedUrls); + + $siteUrlsAfter = API::getInstance()->getSiteUrlsFromId($idsite); + + $shouldHave = $siteUrlsBefore; + sort($shouldHave); + + sort($siteUrlsAfter); + + $this->assertEquals($shouldHave, $siteUrlsAfter); + } + + /** + * case array only duplicate => nothing happens + * + * @group Plugins + */ + public function testAddSiteUrlsNoUrlsToAdd2() + { + $idsite = $this->_addSite(); + + $siteUrlsBefore = API::getInstance()->getSiteUrlsFromId($idsite); + + $toAdd = $siteUrlsBefore; + + $insertedUrls = API::getInstance()->addSiteAliasUrls($idsite, $toAdd); + $this->assertEquals(0, $insertedUrls); + + $siteUrlsAfter = API::getInstance()->getSiteUrlsFromId($idsite); + + $shouldHave = $siteUrlsBefore; + sort($shouldHave); + + sort($siteUrlsAfter); + + $this->assertEquals($shouldHave, $siteUrlsAfter); + } + + /** + * wrong format urls => exception + * + * @group Plugins + */ + public function testAddSiteUrlsWrongUrlsFormat3() + { + try { + $idsite = $this->_addSite(); + $toAdd = array("http:mpigeq"); + API::getInstance()->addSiteAliasUrls($idsite, $toAdd); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * wrong idsite => no exception because simply no access to this resource + * + * @group Plugins + */ + public function testAddSiteUrlsWrongIdSite1() + { + try { + $toAdd = array("http://pigeq.com/test"); + API::getInstance()->addSiteAliasUrls(-1, $toAdd); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * wrong idsite => exception + * + * @group Plugins + */ + public function testAddSiteUrlsWrongIdSite2() + { + try { + $toAdd = array("http://pigeq.com/test"); + API::getInstance()->addSiteAliasUrls(155, $toAdd); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * no Id -> empty array + * + * @group Plugins + */ + public function testGetAllSitesIdNoId() + { + $ids = API::getInstance()->getAllSitesId(); + $this->assertEquals(array(), $ids); + } + + /** + * several Id -> normal array + * + * @group Plugins + */ + public function testGetAllSitesIdSeveralId() + { + $name = "tetq"; + $idsites = array( + API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")), + API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")), + API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")), + API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")), + API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")), + ); + + $ids = API::getInstance()->getAllSitesId(); + $this->assertEquals($idsites, $ids); + } + + /** + * wrong id => exception + * + * @group Plugins + */ + public function testGetSiteFromIdWrongId1() + { + try { + API::getInstance()->getSiteFromId(0); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * wrong id => exception + * + * @group Plugins + */ + public function testGetSiteFromIdWrongId2() + { + try { + API::getInstance()->getSiteFromId("x1"); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * wrong id : no access => exception + * + * @group Plugins + */ + public function testGetSiteFromIdWrongId3() + { + $idsite = API::getInstance()->addSite("site", array("http://piwik.net", "http://piwik.com/test/")); + $this->assertEquals(1, $idsite); + + // set noaccess to site 1 + FakeAccess::setIdSitesView(array(2)); + FakeAccess::setIdSitesAdmin(array()); + + try { + API::getInstance()->getSiteFromId(1); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * normal case + * + * @group Plugins + */ + public function testGetSiteFromIdNormalId() + { + $name = "website ''"; + $idsite = API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")); + $this->assertInternalType(PHPUnit_Framework_Constraint_IsType::TYPE_INT, $idsite); + + $siteInfo = API::getInstance()->getSiteFromId($idsite); + $this->assertEquals($name, $siteInfo['name']); + $this->assertEquals("http://piwik.net", $siteInfo['main_url']); + } + + /** + * there is no admin site available -> array() + * + * @group Plugins + */ + public function testGetSitesWithAdminAccessNoResult() + { + FakeAccess::setIdSitesAdmin(array()); + + $sites = API::getInstance()->getSitesWithAdminAccess(); + $this->assertEquals(array(), $sites); + } + + /** + * normal case, admin and view and noaccess website => return only admin + * + * @group Plugins + */ + public function testGetSitesWithAdminAccess() + { + API::getInstance()->addSite("site1", array("http://piwik.net", "http://piwik.com/test/")); + API::getInstance()->addSite("site2", array("http://piwik.com/test/")); + API::getInstance()->addSite("site3", array("http://piwik.org")); + + $resultWanted = array( + 0 => array("idsite" => 1, "name" => "site1", "main_url" => "http://piwik.net", "ecommerce" => 0, "excluded_ips" => "", 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), + 1 => array("idsite" => 3, "name" => "site3", "main_url" => "http://piwik.org", "ecommerce" => 0, "excluded_ips" => "", 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), + ); + + FakeAccess::setIdSitesAdmin(array(1, 3)); + + $sites = API::getInstance()->getSitesWithAdminAccess(); + + // we dont test the ts_created + unset($sites[0]['ts_created']); + unset($sites[1]['ts_created']); + $this->assertEquals($resultWanted, $sites); + } + + /** + * there is no admin site available -> array() + * + * @group Plugins + */ + public function testGetSitesWithViewAccessNoResult() + { + FakeAccess::setIdSitesView(array()); + FakeAccess::setIdSitesAdmin(array()); + + $sites = API::getInstance()->getSitesWithViewAccess(); + $this->assertEquals(array(), $sites); + } + + /** + * normal case, admin and view and noaccess website => return only admin + * + * @group Plugins + */ + public function testGetSitesWithViewAccess() + { + API::getInstance()->addSite("site1", array("http://piwik.net", "http://piwik.com/test/")); + API::getInstance()->addSite("site2", array("http://piwik.com/test/")); + API::getInstance()->addSite("site3", array("http://piwik.org")); + + $resultWanted = array( + 0 => array("idsite" => 1, "name" => "site1", "main_url" => "http://piwik.net", "ecommerce" => 0, 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', "excluded_ips" => "", 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), + 1 => array("idsite" => 3, "name" => "site3", "main_url" => "http://piwik.org", "ecommerce" => 0, 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', "excluded_ips" => "", 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), + ); + + FakeAccess::setIdSitesView(array(1, 3)); + FakeAccess::setIdSitesAdmin(array()); + + $sites = API::getInstance()->getSitesWithViewAccess(); + // we dont test the ts_created + unset($sites[0]['ts_created']); + unset($sites[1]['ts_created']); + $this->assertEquals($resultWanted, $sites); + } + + /** + * there is no admin site available -> array() + * + * @group Plugins + */ + public function testGetSitesWithAtLeastViewAccessNoResult() + { + FakeAccess::setIdSitesView(array()); + FakeAccess::setIdSitesAdmin(array()); + + $sites = API::getInstance()->getSitesWithAtLeastViewAccess(); + $this->assertEquals(array(), $sites); + } + + /** + * normal case, admin and view and noaccess website => return only admin + * + * @group Plugins + */ + public function testGetSitesWithAtLeastViewAccess() + { + API::getInstance()->addSite("site1", array("http://piwik.net", "http://piwik.com/test/"), $ecommerce = 1); + API::getInstance()->addSite("site2", array("http://piwik.com/test/")); + API::getInstance()->addSite("site3", array("http://piwik.org")); + + $resultWanted = array( + 0 => array("idsite" => 1, "name" => "site1", "main_url" => "http://piwik.net", "ecommerce" => 1, "excluded_ips" => "", 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), + 1 => array("idsite" => 3, "name" => "site3", "main_url" => "http://piwik.org", "ecommerce" => 0, "excluded_ips" => "", 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), + ); + + FakeAccess::setIdSitesView(array(1, 3)); + FakeAccess::setIdSitesAdmin(array()); + + $sites = API::getInstance()->getSitesWithAtLeastViewAccess(); + // we dont test the ts_created + unset($sites[0]['ts_created']); + unset($sites[1]['ts_created']); + $this->assertEquals($resultWanted, $sites); + } + + /** + * no urls for this site => array() + * + * @group Plugins + */ + public function testGetSiteUrlsFromIdNoUrls() + { + $idsite = API::getInstance()->addSite("site1", array("http://piwik.net")); + + $urls = API::getInstance()->getSiteUrlsFromId($idsite); + $this->assertEquals(array("http://piwik.net"), $urls); + } + + /** + * normal case + * + * @group Plugins + */ + public function testGetSiteUrlsFromIdManyUrls() + { + $site = array("http://piwik.net", + "http://piwik.org", + "http://piwik.org", + "http://piwik.com"); + sort($site); + + $idsite = API::getInstance()->addSite("site1", $site); + + $siteWanted = array("http://piwik.net", + "http://piwik.org", + "http://piwik.com"); + sort($siteWanted); + $urls = API::getInstance()->getSiteUrlsFromId($idsite); + + $this->assertEquals($siteWanted, $urls); + } + + /** + * wrongId => exception + * + * @group Plugins + */ + public function testGetSiteUrlsFromIdWrongId() + { + try { + FakeAccess::setIdSitesView(array(3)); + FakeAccess::setIdSitesAdmin(array()); + API::getInstance()->getSiteUrlsFromId(1); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * one url => no change to alias urls + * + * @group Plugins + */ + public function testUpdateSiteOneUrl() + { + $urls = array("http://piwiknew.com", + "http://piwiknew.net", + "http://piwiknew.org", + "http://piwiknew.fr"); + $idsite = API::getInstance()->addSite("site1", $urls); + + $newMainUrl = "http://main.url"; + + // Also test that the group was set to empty, and is searchable + $websites = API::getInstance()->getSitesFromGroup(''); + $this->assertEquals(1, count($websites)); + + // the Update doesn't change the group field + API::getInstance()->updateSite($idsite, "test toto@{}", $newMainUrl); + $websites = API::getInstance()->getSitesFromGroup(''); + $this->assertEquals(1, count($websites)); + + // Updating the group to something + $group = 'something'; + API::getInstance()->updateSite($idsite, "test toto@{}", $newMainUrl, $ecommerce = 0, $ss = true, $ss_kwd = null, $ss_cat = '', $ips = null, $parametersExclude = null, $timezone = null, $currency = null, $group); + $websites = API::getInstance()->getSitesFromGroup($group); + $this->assertEquals(1, count($websites)); + $this->assertEquals(date('Y-m-d'), date('Y-m-d', strtotime($websites[0]['ts_created']))); + + // Updating the group to nothing + $group = ''; + $type = 'mobileAppTest'; + API::getInstance()->updateSite($idsite, "test toto@{}", $newMainUrl, $ecommerce = 0, $ss = false, $ss_kwd = '', $ss_cat = null, $ips = null, $parametersExclude = null, $timezone = null, $currency = null, $group, $startDate = '2010-01-01', $excludedUserAgent = null, $keepUrlFragment = 1, $type); + $websites = API::getInstance()->getSitesFromGroup($group); + $this->assertEquals(1, count($websites)); + $this->assertEquals('2010-01-01', date('Y-m-d', strtotime($websites[0]['ts_created']))); + + // Test setting the website type + $this->assertEquals($type, Site::getTypeFor($idsite)); + + // Check Alias URLs contain only main url + $allUrls = API::getInstance()->getSiteUrlsFromId($idsite); + $this->assertEquals($newMainUrl, $allUrls[0]); + $aliasUrls = array_slice($allUrls, 1); + $this->assertEquals(array(), $aliasUrls); + + } + + /** + * strange name and NO URL => name ok, main_url not updated + * + * @group Plugins + */ + public function testUpdateSiteStrangeNameNoUrl() + { + $idsite = API::getInstance()->addSite("site1", "http://main.url"); + $newName = "test toto@{'786'}"; + + API::getInstance()->updateSite($idsite, $newName); + + $site = API::getInstance()->getSiteFromId($idsite); + + $this->assertEquals($newName, $site['name']); + // url didn't change because parameter url NULL in updateSite + $this->assertEquals("http://main.url", $site['main_url']); + } + + /** + * several urls => both main and alias are updated + * also test the update of group field + * + * @group Plugins + */ + public function testUpdateSiteSeveralUrlsAndGroup() + { + $urls = array("http://piwiknew.com", + "http://piwiknew.net", + "http://piwiknew.org", + "http://piwiknew.fr"); + + $group = 'GROUP Before'; + $idsite = API::getInstance()->addSite("site1", $urls, $ecommerce = 1, + $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, + $excludedIps = null, $excludedQueryParameters = null, $timezone = null, $currency = null, $group, $startDate = '2011-01-01'); + + $websites = API::getInstance()->getSitesFromGroup($group); + $this->assertEquals(1, count($websites)); + + $newurls = array("http://piwiknew2.com", + "http://piwiknew2.net", + "http://piwiknew2.org", + "http://piwiknew2.fr"); + + $groupAfter = ' GROUP After'; + API::getInstance()->updateSite($idsite, "test toto@{}", $newurls, $ecommerce = 0, + $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, + $excludedIps = null, $excludedQueryParameters = null, $timezone = null, $currency = null, $groupAfter); + + // no result for the group before update + $websites = API::getInstance()->getSitesFromGroup($group); + $this->assertEquals(0, count($websites)); + + // Testing that the group was updated properly (and testing that the group value is trimmed before inserted/searched) + $websites = API::getInstance()->getSitesFromGroup($groupAfter . ' '); + $this->assertEquals(1, count($websites)); + $this->assertEquals('2011-01-01', date('Y-m-d', strtotime($websites[0]['ts_created']))); + + // Test fetch website groups + $expectedGroups = array(trim($groupAfter)); + $fetched = API::getInstance()->getSitesGroups(); + $this->assertEquals($expectedGroups, $fetched); + + $allUrls = API::getInstance()->getSiteUrlsFromId($idsite); + sort($allUrls); + sort($newurls); + $this->assertEquals($newurls, $allUrls); + } + + /** + * @expectedException Exception + * @expectedExceptionMessage SitesManager_ExceptionDeleteSite + */ + public function test_delete_ShouldNotDeleteASiteInCaseThereIsOnlyOneSite() + { + $siteId1 = $this->_addSite(); + + $this->assertHasSite($siteId1); + + try { + API::getInstance()->deleteSite($siteId1); + $this->fail('an expected exception was not raised'); + } catch (Exception $e) { + $this->assertHasSite($siteId1); + throw $e; + } + } + + /** + * @expectedException Exception + * @expectedExceptionMessage website id = 99999498 not found + */ + public function test_delete_ShouldTriggerException_IfGivenSiteDoesNotExist() + { + API::getInstance()->deleteSite(99999498); + } + + public function test_delete_ShouldActuallyRemoveAnExistingSiteButOnlyTheGivenSite() + { + $this->_addSite(); + $siteId1 = $this->_addSite(); + $siteId2 = $this->_addSite(); + + $this->assertHasSite($siteId1); + $this->assertHasSite($siteId2); + + API::getInstance()->deleteSite($siteId1); + + $this->assertHasNotSite($siteId1); + $this->assertHasSite($siteId2); + } + + public function test_delete_ShouldTriggerAnEventOnceSiteWasActuallyDeleted() + { + $called = 0; + $deletedSiteId = null; + + Piwik::addAction('SitesManager.deleteSite.end', function ($param) use (&$called, &$deletedSiteId) { + $called++; + $deletedSiteId = $param; + }); + + $this->_addSite(); + $siteId1 = $this->_addSite(); + + API::getInstance()->deleteSite($siteId1); + + $this->assertSame(1, $called); + $this->assertSame($siteId1, $deletedSiteId); + } + + private function assertHasSite($idSite) + { + $model = new Model(); + $siteInfo = $model->getSiteFromId($idSite); + $this->assertNotEmpty($siteInfo); + } + + private function assertHasNotSite($idSite) + { + $model = new Model(); + $siteInfo = $model->getSiteFromId($idSite); + $this->assertEmpty($siteInfo); + } + + /** + * @group Plugins + */ + public function testGetSitesGroups() + { + $groups = array('group1', ' group1 ', '', 'group2'); + $expectedGroups = array('group1', '', 'group2'); + foreach ($groups as $group) { + API::getInstance()->addSite("test toto@{}", 'http://example.org', $ecommerce = 1, $siteSearch = null, $searchKeywordParameters = null, $searchCategoryParameters = null, $excludedIps = null, $excludedQueryParameters = null, $timezone = null, $currency = null, $group); + } + + $this->assertEquals($expectedGroups, API::getInstance()->getSitesGroups()); + } + + public function getInvalidTimezoneData() + { + return array( + array('UTC+15'), + array('Paris'), + ); + } + + /** + * + * @dataProvider getInvalidTimezoneData + * @group Plugins + */ + public function testAddSitesInvalidTimezone($timezone) + { + try { + API::getInstance()->addSite("site1", array('http://example.org'), $ecommerce = 0, + $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, $ip = '', $params = '', $timezone); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * @group Plugins + */ + public function testAddSitesInvalidCurrency() + { + try { + $invalidCurrency = '€'; + API::getInstance()->addSite("site1", array('http://example.org'), $ecommerce = 0, + $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, '', 'UTC', $invalidCurrency); + } catch (Exception $e) { + return; + } + $this->fail('Expected exception not raised'); + } + + /** + * @group Plugins + */ + public function testSetDefaultTimezoneAndCurrencyAndExcludedQueryParametersAndExcludedIps() + { + // test that they return default values + $defaultTimezone = API::getInstance()->getDefaultTimezone(); + $this->assertEquals('UTC', $defaultTimezone); + $defaultCurrency = API::getInstance()->getDefaultCurrency(); + $this->assertEquals('USD', $defaultCurrency); + $excludedIps = API::getInstance()->getExcludedIpsGlobal(); + $this->assertEquals('', $excludedIps); + $excludedQueryParameters = API::getInstance()->getExcludedQueryParametersGlobal(); + $this->assertEquals('', $excludedQueryParameters); + + // test that when not specified, defaults are set as expected + $idsite = API::getInstance()->addSite("site1", array('http://example.org')); + $site = new Site($idsite); + $this->assertEquals('UTC', $site->getTimezone()); + $this->assertEquals('USD', $site->getCurrency()); + $this->assertEquals('', $site->getExcludedQueryParameters()); + $this->assertEquals('', $site->getExcludedIps()); + $this->assertEquals(false, $site->isEcommerceEnabled()); + + // set the global timezone and get it + $newDefaultTimezone = 'UTC+5.5'; + API::getInstance()->setDefaultTimezone($newDefaultTimezone); + $defaultTimezone = API::getInstance()->getDefaultTimezone(); + $this->assertEquals($newDefaultTimezone, $defaultTimezone); + + // set the default currency and get it + $newDefaultCurrency = 'EUR'; + API::getInstance()->setDefaultCurrency($newDefaultCurrency); + $defaultCurrency = API::getInstance()->getDefaultCurrency(); + $this->assertEquals($newDefaultCurrency, $defaultCurrency); + + // set the global IPs to exclude and get it + $newGlobalExcludedIps = '1.1.1.*,1.1.*.*,150.1.1.1'; + API::getInstance()->setGlobalExcludedIps($newGlobalExcludedIps); + $globalExcludedIps = API::getInstance()->getExcludedIpsGlobal(); + $this->assertEquals($newGlobalExcludedIps, $globalExcludedIps); + + // set the global URL query params to exclude and get it + $newGlobalExcludedQueryParameters = 'PHPSESSID,blabla, TesT'; + // removed the space + $expectedGlobalExcludedQueryParameters = 'PHPSESSID,blabla,TesT'; + API::getInstance()->setGlobalExcludedQueryParameters($newGlobalExcludedQueryParameters); + $globalExcludedQueryParameters = API::getInstance()->getExcludedQueryParametersGlobal(); + $this->assertEquals($expectedGlobalExcludedQueryParameters, $globalExcludedQueryParameters); + + // create a website and check that default currency and default timezone are set + // however, excluded IPs and excluded query Params are not returned + $idsite = API::getInstance()->addSite("site1", array('http://example.org'), $ecommerce = 0, + $siteSearch = 0, $searchKeywordParameters = 'test1,test2', $searchCategoryParameters = 'test2,test1', + '', '', $newDefaultTimezone); + $site = new Site($idsite); + $this->assertEquals($newDefaultTimezone, $site->getTimezone()); + $this->assertEquals(date('Y-m-d'), $site->getCreationDate()->toString()); + $this->assertEquals($newDefaultCurrency, $site->getCurrency()); + $this->assertEquals('', $site->getExcludedIps()); + $this->assertEquals('', $site->getExcludedQueryParameters()); + $this->assertEquals('test1,test2', $site->getSearchKeywordParameters()); + $this->assertEquals('test2,test1', $site->getSearchCategoryParameters()); + $this->assertFalse($site->isSiteSearchEnabled()); + $this->assertFalse(Site::isSiteSearchEnabledFor($idsite)); + $this->assertFalse($site->isEcommerceEnabled()); + $this->assertFalse(Site::isEcommerceEnabledFor($idsite)); + } + + /** + * @group Plugins + */ + public function testGetSitesIdFromSiteUrlSuperUser() + { + API::getInstance()->addSite("site1", array("http://piwik.net", "http://piwik.com")); + API::getInstance()->addSite("site2", array("http://piwik.com", "http://piwik.net")); + API::getInstance()->addSite("site3", array("http://piwik.com", "http://piwik.org")); + + $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.org'); + $this->assertTrue(count($idsites) == 1); + + $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://www.piwik.org'); + $this->assertTrue(count($idsites) == 1); + + $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.net'); + $this->assertTrue(count($idsites) == 2); + + $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.com'); + $this->assertTrue(count($idsites) == 3); + } + + /** + * @group Plugins + */ + public function testGetSitesIdFromSiteUrlUser() + { + API::getInstance()->addSite("site1", array("http://www.piwik.net", "http://piwik.com")); + API::getInstance()->addSite("site2", array("http://piwik.com", "http://piwik.net")); + API::getInstance()->addSite("site3", array("http://piwik.com", "http://piwik.org")); + + $saveAccess = Access::getInstance(); + + APIUsersManager::getInstance()->addUser("user1", "geqgegagae", "tegst@tesgt.com", "alias"); + APIUsersManager::getInstance()->setUserAccess("user1", "view", array(1)); + + APIUsersManager::getInstance()->addUser("user2", "geqgegagae", "tegst2@tesgt.com", "alias"); + APIUsersManager::getInstance()->setUserAccess("user2", "view", array(1)); + APIUsersManager::getInstance()->setUserAccess("user2", "admin", array(3)); + + APIUsersManager::getInstance()->addUser("user3", "geqgegagae", "tegst3@tesgt.com", "alias"); + APIUsersManager::getInstance()->setUserAccess("user3", "view", array(1, 2)); + APIUsersManager::getInstance()->setUserAccess("user3", "admin", array(3)); + + $pseudoMockAccess = new FakeAccess; + FakeAccess::$superUser = false; + FakeAccess::$identity = 'user1'; + FakeAccess::setIdSitesView(array(1)); + FakeAccess::setIdSitesAdmin(array()); + Access::setSingletonInstance($pseudoMockAccess); + $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.com'); + $this->assertEquals(1, count($idsites)); + + // testing URL normalization + $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://www.piwik.com'); + $this->assertEquals(1, count($idsites)); + $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.net'); + $this->assertEquals(1, count($idsites)); + + $pseudoMockAccess = new FakeAccess; + FakeAccess::$superUser = false; + FakeAccess::$identity = 'user2'; + FakeAccess::setIdSitesView(array(1)); + FakeAccess::setIdSitesAdmin(array(3)); + Access::setSingletonInstance($pseudoMockAccess); + $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.com'); + $this->assertEquals(2, count($idsites)); + + $pseudoMockAccess = new FakeAccess; + FakeAccess::$superUser = false; + FakeAccess::$identity = 'user3'; + FakeAccess::setIdSitesView(array(1, 2)); + FakeAccess::setIdSitesAdmin(array(3)); + Access::setSingletonInstance($pseudoMockAccess); + $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.com'); + $this->assertEquals(3, count($idsites)); + + Access::setSingletonInstance($saveAccess); + } + + /** + * @group Plugins + */ + public function testGetSitesFromTimezones() + { + API::getInstance()->addSite("site3", array("http://piwik.org"), null, $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, null, null, 'UTC'); + $idsite2 = API::getInstance()->addSite("site3", array("http://piwik.org"), null, $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, null, null, 'Pacific/Auckland'); + $idsite3 = API::getInstance()->addSite("site3", array("http://piwik.org"), null, $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, null, null, 'Pacific/Auckland'); + $idsite4 = API::getInstance()->addSite("site3", array("http://piwik.org"), null, $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, null, null, 'UTC+10'); + $result = API::getInstance()->getSitesIdFromTimezones(array('UTC+10', 'Pacific/Auckland')); + $this->assertEquals(array($idsite2, $idsite3, $idsite4), $result); + } +} diff --git a/plugins/SitesManager/tests/Integration/SitesManagerTest.php b/plugins/SitesManager/tests/Integration/SitesManagerTest.php index 3f82a95574..6722e51bed 100644 --- a/plugins/SitesManager/tests/Integration/SitesManagerTest.php +++ b/plugins/SitesManager/tests/Integration/SitesManagerTest.php @@ -9,13 +9,13 @@ namespace Piwik\Plugins\SitesManager\tests\Integration; use Piwik\Access; -use Piwik\Plugins\SitesManager\API; -use Piwik\Plugins\UsersManager\API as APIUsersManager; -use Piwik\Site; +use Piwik\Cache; +use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Date; +use Piwik\Plugins\SitesManager\SitesManager; +use Piwik\Tests\Fixture; use Piwik\Tests\Framework\Mock\FakeAccess; use Piwik\Tests\Framework\TestCase\IntegrationTestCase; -use Exception; -use PHPUnit_Framework_Constraint_IsType; /** * Class Plugins_SitesManagerTest @@ -24,6 +24,13 @@ use PHPUnit_Framework_Constraint_IsType; */ class SitesManagerTest extends IntegrationTestCase { + /** + * @var SitesManager + */ + private $manager; + + private $siteId; + public function setUp() { parent::setUp(); @@ -32,969 +39,41 @@ class SitesManagerTest extends IntegrationTestCase $pseudoMockAccess = new FakeAccess; FakeAccess::$superUser = true; Access::setSingletonInstance($pseudoMockAccess); - } - - /** - * empty name -> exception - * - * @group Plugins - */ - public function testAddSiteEmptyName() - { - try { - API::getInstance()->addSite("", array("http://piwik.net")); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); - } - - /** - * DataProvider for testAddSiteWrongUrls - */ - public function getInvalidUrlData() - { - return array( - array(array()), // no urls - array(array("")), - array(""), - array("httpww://piwik.net"), - array("httpww://piwik.net/gqg~#"), - ); - } - - /** - * wrong urls -> exception - * - * @dataProvider getInvalidUrlData - * @group Plugins - */ - public function testAddSiteWrongUrls($url) - { - try { - API::getInstance()->addSite("name", $url); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); - } - - /** - * Test with valid IPs - * - * @group Plugins - */ - public function testAddSiteExcludedIpsAndtimezoneAndCurrencyAndExcludedQueryParametersValid() - { - $ips = '1.2.3.4,1.1.1.*,1.2.*.*,1.*.*.*'; - $timezone = 'Europe/Paris'; - $currency = 'EUR'; - $excludedQueryParameters = 'p1,P2, P33333'; - $expectedExcludedQueryParameters = 'p1,P2,P33333'; - $excludedUserAgents = " p1,P2, \nP3333 "; - $expectedExcludedUserAgents = "p1,P2,P3333"; - $expectedWebsiteType = 'mobile-\'app'; - $keepUrlFragment = 1; - $idsite = API::getInstance()->addSite("name", "http://piwik.net/", $ecommerce = 1, - $siteSearch = 1, $searchKeywordParameters = 'search,param', $searchCategoryParameters = 'cat,category', - $ips, $excludedQueryParameters, $timezone, $currency, $group = null, $startDate = null, $excludedUserAgents, - $keepUrlFragment, $expectedWebsiteType); - $siteInfo = API::getInstance()->getSiteFromId($idsite); - $this->assertEquals($ips, $siteInfo['excluded_ips']); - $this->assertEquals($timezone, $siteInfo['timezone']); - $this->assertEquals($currency, $siteInfo['currency']); - $this->assertEquals($ecommerce, $siteInfo['ecommerce']); - $this->assertTrue(Site::isEcommerceEnabledFor($idsite)); - $this->assertEquals($siteSearch, $siteInfo['sitesearch']); - $this->assertTrue(Site::isSiteSearchEnabledFor($idsite)); - $this->assertEquals($expectedWebsiteType, $siteInfo['type']); - $this->assertEquals($expectedWebsiteType, Site::getTypeFor($idsite)); - - $this->assertEquals($searchKeywordParameters, $siteInfo['sitesearch_keyword_parameters']); - $this->assertEquals($searchCategoryParameters, $siteInfo['sitesearch_category_parameters']); - $this->assertEquals($expectedExcludedQueryParameters, $siteInfo['excluded_parameters']); - $this->assertEquals($expectedExcludedUserAgents, $siteInfo['excluded_user_agents']); - } - - /** - * dataProvider for testAddSiteExcludedIpsNotValid - */ - public function getInvalidIPsData() - { - return array( - array('35817587341'), - array('ieagieha'), - array('1.2.3'), - array('*.1.1.1'), - array('*.*.1.1'), - array('*.*.*.1'), - array('1.1.1.1.1'), - ); - } - /** - * Test with invalid IPs - * - * @dataProvider getInvalidIPsData - * @group Plugins - */ - public function testAddSiteExcludedIpsNotValid($ip) - { - try { - API::getInstance()->addSite("name", "http://piwik.net/", $ecommerce = 0, - $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, $ip); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); + $this->manager = new SitesManager(); + $this->siteId = Fixture::createWebsite('2014-03-03 00:00:00'); } - /** - * one url -> one main_url and nothing inserted as alias urls - * - * @group Plugins - */ - public function testAddSiteOneUrl() + public function test_onSiteDeleted_shouldClearSiteCache() { - $url = "http://piwik.net/"; - $urlOK = "http://piwik.net"; - $idsite = API::getInstance()->addSite("name", $url); - $this->assertInternalType(PHPUnit_Framework_Constraint_IsType::TYPE_INT, $idsite); + $cache = Cache::getLazyCache(); + $cache->save($this->siteId, 'testcontent'); - $siteInfo = API::getInstance()->getSiteFromId($idsite); - $this->assertEquals($urlOK, $siteInfo['main_url']); - $this->assertEquals(date('Y-m-d'), date('Y-m-d', strtotime($siteInfo['ts_created']))); + $this->manager->onSiteDeleted($this->siteId); - $siteUrls = API::getInstance()->getSiteUrlsFromId($idsite); - $this->assertEquals(1, count($siteUrls)); + $this->assertFalse($cache->contains($this->siteId)); } - /** - * several urls -> one main_url and others as alias urls - * - * @group Plugins - */ - public function testAddSiteSeveralUrls() + public function test_onSiteDeleted_shouldRemoveRememberedArchiveReports() { - $urls = array("http://piwik.net/", "http://piwik.com", "https://piwik.net/test/"); - $urlsOK = array("http://piwik.net", "http://piwik.com", "https://piwik.net/test"); - $idsite = API::getInstance()->addSite("super website", $urls); - $this->assertInternalType(PHPUnit_Framework_Constraint_IsType::TYPE_INT, $idsite); + $archive = new ArchiveInvalidator(); + $archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-05')); + $archive->rememberToInvalidateArchivedReportsLater($this->siteId, Date::factory('2014-04-06')); + $archive->rememberToInvalidateArchivedReportsLater(4949, Date::factory('2014-04-05')); - $siteInfo = API::getInstance()->getSiteFromId($idsite); - $this->assertEquals($urlsOK[0], $siteInfo['main_url']); - - $siteUrls = API::getInstance()->getSiteUrlsFromId($idsite); - $this->assertEquals($urlsOK, $siteUrls); - } - - /** - * strange name - * - * @group Plugins - */ - public function testAddSiteStrangeName() - { - $name = "supertest(); ~@@()''!£\$'%%^'!£ போ"; - $idsite = API::getInstance()->addSite($name, "http://piwik.net"); - $this->assertInternalType(PHPUnit_Framework_Constraint_IsType::TYPE_INT, $idsite); - - $siteInfo = API::getInstance()->getSiteFromId($idsite); - $this->assertEquals($name, $siteInfo['name']); - - } - - /** - * adds a site - * use by several other unit tests - */ - protected function _addSite() - { - $name = "website "; - $idsite = API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")); - $this->assertInternalType(PHPUnit_Framework_Constraint_IsType::TYPE_INT, $idsite); - - $siteInfo = API::getInstance()->getSiteFromId($idsite); - $this->assertEquals($name, $siteInfo['name']); - $this->assertEquals("http://piwik.net", $siteInfo['main_url']); - - $siteUrls = API::getInstance()->getSiteUrlsFromId($idsite); - $this->assertEquals(array("http://piwik.net", "http://piwik.com/test"), $siteUrls); - - return $idsite; - } - - /** - * no duplicate -> all the urls are saved - * - * @group Plugins - */ - public function testAddSiteUrlsnoDuplicate() - { - $idsite = $this->_addSite(); - - $siteUrlsBefore = API::getInstance()->getSiteUrlsFromId($idsite); - - $toAdd = array("http://piwik1.net", - "http://piwik2.net", - "http://piwik3.net/test/", - "http://localhost/test", - "http://localho5.st/test", - "http://l42578gqege.f4", - "http://super.com/test/test/atqata675675/te" + $expected = array( + '2014-04-05' => array($this->siteId, 4949), + '2014-04-06' => array($this->siteId) ); - $toAddValid = array("http://piwik1.net", - "http://piwik2.net", - "http://piwik3.net/test", - "http://localhost/test", - "http://localho5.st/test", - "http://l42578gqege.f4", - "http://super.com/test/test/atqata675675/te"); - - $insertedUrls = API::getInstance()->addSiteAliasUrls($idsite, $toAdd); - $this->assertEquals(count($toAdd), $insertedUrls); - - $siteUrlsAfter = API::getInstance()->getSiteUrlsFromId($idsite); - $shouldHave = array_merge($siteUrlsBefore, $toAddValid); - sort($shouldHave); + $this->assertEquals($expected, $archive->getRememberedArchivedReportsThatShouldBeInvalidated()); - sort($siteUrlsAfter); - - $this->assertEquals($shouldHave, $siteUrlsAfter); - } - - /** - * duplicate -> don't save the already existing URLs - * - * @group Plugins - */ - public function testAddSiteUrlsDuplicate() - { - $idsite = $this->_addSite(); - - $siteUrlsBefore = API::getInstance()->getSiteUrlsFromId($idsite); - - $toAdd = array_merge($siteUrlsBefore, array("http://piwik1.net", "http://piwik2.net")); - - $insertedUrls = API::getInstance()->addSiteAliasUrls($idsite, $toAdd); - $this->assertEquals(count($toAdd) - count($siteUrlsBefore), $insertedUrls); - - $siteUrlsAfter = API::getInstance()->getSiteUrlsFromId($idsite); - - $shouldHave = $toAdd; - sort($shouldHave); - - sort($siteUrlsAfter); - - $this->assertEquals($shouldHave, $siteUrlsAfter); - } + $this->manager->onSiteDeleted($this->siteId); - /** - * case empty array => nothing happens - * - * @group Plugins - */ - public function testAddSiteUrlsNoUrlsToAdd1() - { - $idsite = $this->_addSite(); - - $siteUrlsBefore = API::getInstance()->getSiteUrlsFromId($idsite); - - $toAdd = array(); - - $insertedUrls = API::getInstance()->addSiteAliasUrls($idsite, $toAdd); - $this->assertEquals(count($toAdd), $insertedUrls); - - $siteUrlsAfter = API::getInstance()->getSiteUrlsFromId($idsite); - - $shouldHave = $siteUrlsBefore; - sort($shouldHave); - - sort($siteUrlsAfter); - - $this->assertEquals($shouldHave, $siteUrlsAfter); - } - - /** - * case array only duplicate => nothing happens - * - * @group Plugins - */ - public function testAddSiteUrlsNoUrlsToAdd2() - { - $idsite = $this->_addSite(); - - $siteUrlsBefore = API::getInstance()->getSiteUrlsFromId($idsite); - - $toAdd = $siteUrlsBefore; - - $insertedUrls = API::getInstance()->addSiteAliasUrls($idsite, $toAdd); - $this->assertEquals(0, $insertedUrls); - - $siteUrlsAfter = API::getInstance()->getSiteUrlsFromId($idsite); - - $shouldHave = $siteUrlsBefore; - sort($shouldHave); - - sort($siteUrlsAfter); - - $this->assertEquals($shouldHave, $siteUrlsAfter); - } - - /** - * wrong format urls => exception - * - * @group Plugins - */ - public function testAddSiteUrlsWrongUrlsFormat3() - { - try { - $idsite = $this->_addSite(); - $toAdd = array("http:mpigeq"); - API::getInstance()->addSiteAliasUrls($idsite, $toAdd); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); - } - - /** - * wrong idsite => no exception because simply no access to this resource - * - * @group Plugins - */ - public function testAddSiteUrlsWrongIdSite1() - { - try { - $toAdd = array("http://pigeq.com/test"); - API::getInstance()->addSiteAliasUrls(-1, $toAdd); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); - } - - /** - * wrong idsite => exception - * - * @group Plugins - */ - public function testAddSiteUrlsWrongIdSite2() - { - try { - $toAdd = array("http://pigeq.com/test"); - API::getInstance()->addSiteAliasUrls(155, $toAdd); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); - } - - /** - * no Id -> empty array - * - * @group Plugins - */ - public function testGetAllSitesIdNoId() - { - $ids = API::getInstance()->getAllSitesId(); - $this->assertEquals(array(), $ids); - } - - /** - * several Id -> normal array - * - * @group Plugins - */ - public function testGetAllSitesIdSeveralId() - { - $name = "tetq"; - $idsites = array( - API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")), - API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")), - API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")), - API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")), - API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")), + $expected = array( + '2014-04-05' => array(4949) ); - - $ids = API::getInstance()->getAllSitesId(); - $this->assertEquals($idsites, $ids); - } - - /** - * wrong id => exception - * - * @group Plugins - */ - public function testGetSiteFromIdWrongId1() - { - try { - API::getInstance()->getSiteFromId(0); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); - } - - /** - * wrong id => exception - * - * @group Plugins - */ - public function testGetSiteFromIdWrongId2() - { - try { - API::getInstance()->getSiteFromId("x1"); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); + $this->assertEquals($expected, $archive->getRememberedArchivedReportsThatShouldBeInvalidated()); } - /** - * wrong id : no access => exception - * - * @group Plugins - */ - public function testGetSiteFromIdWrongId3() - { - $idsite = API::getInstance()->addSite("site", array("http://piwik.net", "http://piwik.com/test/")); - $this->assertEquals(1, $idsite); - - // set noaccess to site 1 - FakeAccess::setIdSitesView(array(2)); - FakeAccess::setIdSitesAdmin(array()); - - try { - API::getInstance()->getSiteFromId(1); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); - } - - /** - * normal case - * - * @group Plugins - */ - public function testGetSiteFromIdNormalId() - { - $name = "website ''"; - $idsite = API::getInstance()->addSite($name, array("http://piwik.net", "http://piwik.com/test/")); - $this->assertInternalType(PHPUnit_Framework_Constraint_IsType::TYPE_INT, $idsite); - - $siteInfo = API::getInstance()->getSiteFromId($idsite); - $this->assertEquals($name, $siteInfo['name']); - $this->assertEquals("http://piwik.net", $siteInfo['main_url']); - } - - /** - * there is no admin site available -> array() - * - * @group Plugins - */ - public function testGetSitesWithAdminAccessNoResult() - { - FakeAccess::setIdSitesAdmin(array()); - - $sites = API::getInstance()->getSitesWithAdminAccess(); - $this->assertEquals(array(), $sites); - } - - /** - * normal case, admin and view and noaccess website => return only admin - * - * @group Plugins - */ - public function testGetSitesWithAdminAccess() - { - API::getInstance()->addSite("site1", array("http://piwik.net", "http://piwik.com/test/")); - API::getInstance()->addSite("site2", array("http://piwik.com/test/")); - API::getInstance()->addSite("site3", array("http://piwik.org")); - - $resultWanted = array( - 0 => array("idsite" => 1, "name" => "site1", "main_url" => "http://piwik.net", "ecommerce" => 0, "excluded_ips" => "", 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), - 1 => array("idsite" => 3, "name" => "site3", "main_url" => "http://piwik.org", "ecommerce" => 0, "excluded_ips" => "", 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), - ); - - FakeAccess::setIdSitesAdmin(array(1, 3)); - - $sites = API::getInstance()->getSitesWithAdminAccess(); - - // we dont test the ts_created - unset($sites[0]['ts_created']); - unset($sites[1]['ts_created']); - $this->assertEquals($resultWanted, $sites); - } - - /** - * there is no admin site available -> array() - * - * @group Plugins - */ - public function testGetSitesWithViewAccessNoResult() - { - FakeAccess::setIdSitesView(array()); - FakeAccess::setIdSitesAdmin(array()); - - $sites = API::getInstance()->getSitesWithViewAccess(); - $this->assertEquals(array(), $sites); - } - - /** - * normal case, admin and view and noaccess website => return only admin - * - * @group Plugins - */ - public function testGetSitesWithViewAccess() - { - API::getInstance()->addSite("site1", array("http://piwik.net", "http://piwik.com/test/")); - API::getInstance()->addSite("site2", array("http://piwik.com/test/")); - API::getInstance()->addSite("site3", array("http://piwik.org")); - - $resultWanted = array( - 0 => array("idsite" => 1, "name" => "site1", "main_url" => "http://piwik.net", "ecommerce" => 0, 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', "excluded_ips" => "", 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), - 1 => array("idsite" => 3, "name" => "site3", "main_url" => "http://piwik.org", "ecommerce" => 0, 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', "excluded_ips" => "", 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), - ); - - FakeAccess::setIdSitesView(array(1, 3)); - FakeAccess::setIdSitesAdmin(array()); - - $sites = API::getInstance()->getSitesWithViewAccess(); - // we dont test the ts_created - unset($sites[0]['ts_created']); - unset($sites[1]['ts_created']); - $this->assertEquals($resultWanted, $sites); - } - - /** - * there is no admin site available -> array() - * - * @group Plugins - */ - public function testGetSitesWithAtLeastViewAccessNoResult() - { - FakeAccess::setIdSitesView(array()); - FakeAccess::setIdSitesAdmin(array()); - - $sites = API::getInstance()->getSitesWithAtLeastViewAccess(); - $this->assertEquals(array(), $sites); - } - - /** - * normal case, admin and view and noaccess website => return only admin - * - * @group Plugins - */ - public function testGetSitesWithAtLeastViewAccess() - { - API::getInstance()->addSite("site1", array("http://piwik.net", "http://piwik.com/test/"), $ecommerce = 1); - API::getInstance()->addSite("site2", array("http://piwik.com/test/")); - API::getInstance()->addSite("site3", array("http://piwik.org")); - - $resultWanted = array( - 0 => array("idsite" => 1, "name" => "site1", "main_url" => "http://piwik.net", "ecommerce" => 1, "excluded_ips" => "", 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), - 1 => array("idsite" => 3, "name" => "site3", "main_url" => "http://piwik.org", "ecommerce" => 0, "excluded_ips" => "", 'sitesearch' => 1, 'sitesearch_keyword_parameters' => '', 'sitesearch_category_parameters' => '', 'excluded_parameters' => '', 'excluded_user_agents' => '', 'timezone' => 'UTC', 'currency' => 'USD', 'group' => '', 'keep_url_fragment' => 0, 'type' => 'website'), - ); - - FakeAccess::setIdSitesView(array(1, 3)); - FakeAccess::setIdSitesAdmin(array()); - - $sites = API::getInstance()->getSitesWithAtLeastViewAccess(); - // we dont test the ts_created - unset($sites[0]['ts_created']); - unset($sites[1]['ts_created']); - $this->assertEquals($resultWanted, $sites); - } - - /** - * no urls for this site => array() - * - * @group Plugins - */ - public function testGetSiteUrlsFromIdNoUrls() - { - $idsite = API::getInstance()->addSite("site1", array("http://piwik.net")); - - $urls = API::getInstance()->getSiteUrlsFromId($idsite); - $this->assertEquals(array("http://piwik.net"), $urls); - } - - /** - * normal case - * - * @group Plugins - */ - public function testGetSiteUrlsFromIdManyUrls() - { - $site = array("http://piwik.net", - "http://piwik.org", - "http://piwik.org", - "http://piwik.com"); - sort($site); - - $idsite = API::getInstance()->addSite("site1", $site); - - $siteWanted = array("http://piwik.net", - "http://piwik.org", - "http://piwik.com"); - sort($siteWanted); - $urls = API::getInstance()->getSiteUrlsFromId($idsite); - - $this->assertEquals($siteWanted, $urls); - } - - /** - * wrongId => exception - * - * @group Plugins - */ - public function testGetSiteUrlsFromIdWrongId() - { - try { - FakeAccess::setIdSitesView(array(3)); - FakeAccess::setIdSitesAdmin(array()); - API::getInstance()->getSiteUrlsFromId(1); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); - } - - /** - * one url => no change to alias urls - * - * @group Plugins - */ - public function testUpdateSiteOneUrl() - { - $urls = array("http://piwiknew.com", - "http://piwiknew.net", - "http://piwiknew.org", - "http://piwiknew.fr"); - $idsite = API::getInstance()->addSite("site1", $urls); - - $newMainUrl = "http://main.url"; - - // Also test that the group was set to empty, and is searchable - $websites = API::getInstance()->getSitesFromGroup(''); - $this->assertEquals(1, count($websites)); - - // the Update doesn't change the group field - API::getInstance()->updateSite($idsite, "test toto@{}", $newMainUrl); - $websites = API::getInstance()->getSitesFromGroup(''); - $this->assertEquals(1, count($websites)); - - // Updating the group to something - $group = 'something'; - API::getInstance()->updateSite($idsite, "test toto@{}", $newMainUrl, $ecommerce = 0, $ss = true, $ss_kwd = null, $ss_cat = '', $ips = null, $parametersExclude = null, $timezone = null, $currency = null, $group); - $websites = API::getInstance()->getSitesFromGroup($group); - $this->assertEquals(1, count($websites)); - $this->assertEquals(date('Y-m-d'), date('Y-m-d', strtotime($websites[0]['ts_created']))); - - // Updating the group to nothing - $group = ''; - $type = 'mobileAppTest'; - API::getInstance()->updateSite($idsite, "test toto@{}", $newMainUrl, $ecommerce = 0, $ss = false, $ss_kwd = '', $ss_cat = null, $ips = null, $parametersExclude = null, $timezone = null, $currency = null, $group, $startDate = '2010-01-01', $excludedUserAgent = null, $keepUrlFragment = 1, $type); - $websites = API::getInstance()->getSitesFromGroup($group); - $this->assertEquals(1, count($websites)); - $this->assertEquals('2010-01-01', date('Y-m-d', strtotime($websites[0]['ts_created']))); - - // Test setting the website type - $this->assertEquals($type, Site::getTypeFor($idsite)); - - // Check Alias URLs contain only main url - $allUrls = API::getInstance()->getSiteUrlsFromId($idsite); - $this->assertEquals($newMainUrl, $allUrls[0]); - $aliasUrls = array_slice($allUrls, 1); - $this->assertEquals(array(), $aliasUrls); - - } - - /** - * strange name and NO URL => name ok, main_url not updated - * - * @group Plugins - */ - public function testUpdateSiteStrangeNameNoUrl() - { - $idsite = API::getInstance()->addSite("site1", "http://main.url"); - $newName = "test toto@{'786'}"; - - API::getInstance()->updateSite($idsite, $newName); - - $site = API::getInstance()->getSiteFromId($idsite); - - $this->assertEquals($newName, $site['name']); - // url didn't change because parameter url NULL in updateSite - $this->assertEquals("http://main.url", $site['main_url']); - } - - /** - * several urls => both main and alias are updated - * also test the update of group field - * - * @group Plugins - */ - public function testUpdateSiteSeveralUrlsAndGroup() - { - $urls = array("http://piwiknew.com", - "http://piwiknew.net", - "http://piwiknew.org", - "http://piwiknew.fr"); - - $group = 'GROUP Before'; - $idsite = API::getInstance()->addSite("site1", $urls, $ecommerce = 1, - $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, - $excludedIps = null, $excludedQueryParameters = null, $timezone = null, $currency = null, $group, $startDate = '2011-01-01'); - - $websites = API::getInstance()->getSitesFromGroup($group); - $this->assertEquals(1, count($websites)); - - $newurls = array("http://piwiknew2.com", - "http://piwiknew2.net", - "http://piwiknew2.org", - "http://piwiknew2.fr"); - - $groupAfter = ' GROUP After'; - API::getInstance()->updateSite($idsite, "test toto@{}", $newurls, $ecommerce = 0, - $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, - $excludedIps = null, $excludedQueryParameters = null, $timezone = null, $currency = null, $groupAfter); - - // no result for the group before update - $websites = API::getInstance()->getSitesFromGroup($group); - $this->assertEquals(0, count($websites)); - - // Testing that the group was updated properly (and testing that the group value is trimmed before inserted/searched) - $websites = API::getInstance()->getSitesFromGroup($groupAfter . ' '); - $this->assertEquals(1, count($websites)); - $this->assertEquals('2011-01-01', date('Y-m-d', strtotime($websites[0]['ts_created']))); - - // Test fetch website groups - $expectedGroups = array(trim($groupAfter)); - $fetched = API::getInstance()->getSitesGroups(); - $this->assertEquals($expectedGroups, $fetched); - - $allUrls = API::getInstance()->getSiteUrlsFromId($idsite); - sort($allUrls); - sort($newurls); - $this->assertEquals($newurls, $allUrls); - } - - /** - * @group Plugins - */ - public function testGetSitesGroups() - { - $groups = array('group1', ' group1 ', '', 'group2'); - $expectedGroups = array('group1', '', 'group2'); - foreach ($groups as $group) { - API::getInstance()->addSite("test toto@{}", 'http://example.org', $ecommerce = 1, $siteSearch = null, $searchKeywordParameters = null, $searchCategoryParameters = null, $excludedIps = null, $excludedQueryParameters = null, $timezone = null, $currency = null, $group); - } - - $this->assertEquals($expectedGroups, API::getInstance()->getSitesGroups()); - } - - public function getInvalidTimezoneData() - { - return array( - array('UTC+15'), - array('Paris'), - ); - } - - /** - * - * @dataProvider getInvalidTimezoneData - * @group Plugins - */ - public function testAddSitesInvalidTimezone($timezone) - { - try { - API::getInstance()->addSite("site1", array('http://example.org'), $ecommerce = 0, - $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, $ip = '', $params = '', $timezone); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); - } - - /** - * @group Plugins - */ - public function testAddSitesInvalidCurrency() - { - try { - $invalidCurrency = '€'; - API::getInstance()->addSite("site1", array('http://example.org'), $ecommerce = 0, - $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, '', 'UTC', $invalidCurrency); - } catch (Exception $e) { - return; - } - $this->fail('Expected exception not raised'); - } - - /** - * @group Plugins - */ - public function testSetDefaultTimezoneAndCurrencyAndExcludedQueryParametersAndExcludedIps() - { - // test that they return default values - $defaultTimezone = API::getInstance()->getDefaultTimezone(); - $this->assertEquals('UTC', $defaultTimezone); - $defaultCurrency = API::getInstance()->getDefaultCurrency(); - $this->assertEquals('USD', $defaultCurrency); - $excludedIps = API::getInstance()->getExcludedIpsGlobal(); - $this->assertEquals('', $excludedIps); - $excludedQueryParameters = API::getInstance()->getExcludedQueryParametersGlobal(); - $this->assertEquals('', $excludedQueryParameters); - - // test that when not specified, defaults are set as expected - $idsite = API::getInstance()->addSite("site1", array('http://example.org')); - $site = new Site($idsite); - $this->assertEquals('UTC', $site->getTimezone()); - $this->assertEquals('USD', $site->getCurrency()); - $this->assertEquals('', $site->getExcludedQueryParameters()); - $this->assertEquals('', $site->getExcludedIps()); - $this->assertEquals(false, $site->isEcommerceEnabled()); - - // set the global timezone and get it - $newDefaultTimezone = 'UTC+5.5'; - API::getInstance()->setDefaultTimezone($newDefaultTimezone); - $defaultTimezone = API::getInstance()->getDefaultTimezone(); - $this->assertEquals($newDefaultTimezone, $defaultTimezone); - - // set the default currency and get it - $newDefaultCurrency = 'EUR'; - API::getInstance()->setDefaultCurrency($newDefaultCurrency); - $defaultCurrency = API::getInstance()->getDefaultCurrency(); - $this->assertEquals($newDefaultCurrency, $defaultCurrency); - - // set the global IPs to exclude and get it - $newGlobalExcludedIps = '1.1.1.*,1.1.*.*,150.1.1.1'; - API::getInstance()->setGlobalExcludedIps($newGlobalExcludedIps); - $globalExcludedIps = API::getInstance()->getExcludedIpsGlobal(); - $this->assertEquals($newGlobalExcludedIps, $globalExcludedIps); - - // set the global URL query params to exclude and get it - $newGlobalExcludedQueryParameters = 'PHPSESSID,blabla, TesT'; - // removed the space - $expectedGlobalExcludedQueryParameters = 'PHPSESSID,blabla,TesT'; - API::getInstance()->setGlobalExcludedQueryParameters($newGlobalExcludedQueryParameters); - $globalExcludedQueryParameters = API::getInstance()->getExcludedQueryParametersGlobal(); - $this->assertEquals($expectedGlobalExcludedQueryParameters, $globalExcludedQueryParameters); - - // create a website and check that default currency and default timezone are set - // however, excluded IPs and excluded query Params are not returned - $idsite = API::getInstance()->addSite("site1", array('http://example.org'), $ecommerce = 0, - $siteSearch = 0, $searchKeywordParameters = 'test1,test2', $searchCategoryParameters = 'test2,test1', - '', '', $newDefaultTimezone); - $site = new Site($idsite); - $this->assertEquals($newDefaultTimezone, $site->getTimezone()); - $this->assertEquals(date('Y-m-d'), $site->getCreationDate()->toString()); - $this->assertEquals($newDefaultCurrency, $site->getCurrency()); - $this->assertEquals('', $site->getExcludedIps()); - $this->assertEquals('', $site->getExcludedQueryParameters()); - $this->assertEquals('test1,test2', $site->getSearchKeywordParameters()); - $this->assertEquals('test2,test1', $site->getSearchCategoryParameters()); - $this->assertFalse($site->isSiteSearchEnabled()); - $this->assertFalse(Site::isSiteSearchEnabledFor($idsite)); - $this->assertFalse($site->isEcommerceEnabled()); - $this->assertFalse(Site::isEcommerceEnabledFor($idsite)); - } - - /** - * @group Plugins - */ - public function testGetSitesIdFromSiteUrlSuperUser() - { - API::getInstance()->addSite("site1", array("http://piwik.net", "http://piwik.com")); - API::getInstance()->addSite("site2", array("http://piwik.com", "http://piwik.net")); - API::getInstance()->addSite("site3", array("http://piwik.com", "http://piwik.org")); - - $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.org'); - $this->assertTrue(count($idsites) == 1); - - $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://www.piwik.org'); - $this->assertTrue(count($idsites) == 1); - - $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.net'); - $this->assertTrue(count($idsites) == 2); - - $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.com'); - $this->assertTrue(count($idsites) == 3); - } - - /** - * @group Plugins - */ - public function testGetSitesIdFromSiteUrlUser() - { - API::getInstance()->addSite("site1", array("http://www.piwik.net", "http://piwik.com")); - API::getInstance()->addSite("site2", array("http://piwik.com", "http://piwik.net")); - API::getInstance()->addSite("site3", array("http://piwik.com", "http://piwik.org")); - - $saveAccess = Access::getInstance(); - - APIUsersManager::getInstance()->addUser("user1", "geqgegagae", "tegst@tesgt.com", "alias"); - APIUsersManager::getInstance()->setUserAccess("user1", "view", array(1)); - - APIUsersManager::getInstance()->addUser("user2", "geqgegagae", "tegst2@tesgt.com", "alias"); - APIUsersManager::getInstance()->setUserAccess("user2", "view", array(1)); - APIUsersManager::getInstance()->setUserAccess("user2", "admin", array(3)); - - APIUsersManager::getInstance()->addUser("user3", "geqgegagae", "tegst3@tesgt.com", "alias"); - APIUsersManager::getInstance()->setUserAccess("user3", "view", array(1, 2)); - APIUsersManager::getInstance()->setUserAccess("user3", "admin", array(3)); - - $pseudoMockAccess = new FakeAccess; - FakeAccess::$superUser = false; - FakeAccess::$identity = 'user1'; - FakeAccess::setIdSitesView(array(1)); - FakeAccess::setIdSitesAdmin(array()); - Access::setSingletonInstance($pseudoMockAccess); - $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.com'); - $this->assertEquals(1, count($idsites)); - - // testing URL normalization - $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://www.piwik.com'); - $this->assertEquals(1, count($idsites)); - $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.net'); - $this->assertEquals(1, count($idsites)); - - $pseudoMockAccess = new FakeAccess; - FakeAccess::$superUser = false; - FakeAccess::$identity = 'user2'; - FakeAccess::setIdSitesView(array(1)); - FakeAccess::setIdSitesAdmin(array(3)); - Access::setSingletonInstance($pseudoMockAccess); - $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.com'); - $this->assertEquals(2, count($idsites)); - - $pseudoMockAccess = new FakeAccess; - FakeAccess::$superUser = false; - FakeAccess::$identity = 'user3'; - FakeAccess::setIdSitesView(array(1, 2)); - FakeAccess::setIdSitesAdmin(array(3)); - Access::setSingletonInstance($pseudoMockAccess); - $idsites = API::getInstance()->getSitesIdFromSiteUrl('http://piwik.com'); - $this->assertEquals(3, count($idsites)); - - Access::setSingletonInstance($saveAccess); - } - - /** - * @group Plugins - */ - public function testGetSitesFromTimezones() - { - API::getInstance()->addSite("site3", array("http://piwik.org"), null, $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, null, null, 'UTC'); - $idsite2 = API::getInstance()->addSite("site3", array("http://piwik.org"), null, $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, null, null, 'Pacific/Auckland'); - $idsite3 = API::getInstance()->addSite("site3", array("http://piwik.org"), null, $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, null, null, 'Pacific/Auckland'); - $idsite4 = API::getInstance()->addSite("site3", array("http://piwik.org"), null, $siteSearch = 1, $searchKeywordParameters = null, $searchCategoryParameters = null, null, null, 'UTC+10'); - $result = API::getInstance()->getSitesIdFromTimezones(array('UTC+10', 'Pacific/Auckland')); - $this->assertEquals(array($idsite2, $idsite3, $idsite4), $result); - } } diff --git a/tests/PHPUnit/Framework/TestCase/SystemTestCase.php b/tests/PHPUnit/Framework/TestCase/SystemTestCase.php index 3c8061fef0..6429f5cf4f 100755 --- a/tests/PHPUnit/Framework/TestCase/SystemTestCase.php +++ b/tests/PHPUnit/Framework/TestCase/SystemTestCase.php @@ -46,6 +46,9 @@ abstract class SystemTestCase extends PHPUnit_Framework_TestCase protected $missingExpectedFiles = array(); protected $comparisonFailures = array(); + /** + * @var Fixture + */ public static $fixture; public static function setUpBeforeClass() diff --git a/tests/PHPUnit/Integration/CronArchiveTest.php b/tests/PHPUnit/Integration/CronArchiveTest.php new file mode 100644 index 0000000000..a85f227802 --- /dev/null +++ b/tests/PHPUnit/Integration/CronArchiveTest.php @@ -0,0 +1,69 @@ +rememberToInvalidateArchivedReportsLater(1, Date::factory('2014-04-05')); + $ar->rememberToInvalidateArchivedReportsLater(2, Date::factory('2014-04-05')); + $ar->rememberToInvalidateArchivedReportsLater(2, Date::factory('2014-04-06')); + + $api = API::getInstance(); + + ob_start(); + $cronarchive = new TestCronArchive(Fixture::getRootUrl() . 'tests/PHPUnit/proxy/index.php'); + $cronarchive->setApiToInvalidateArchivedReport($api); + $cronarchive->init(); + ob_end_clean(); + + $expectedInvalidations = array( + array(array(1,2), '2014-04-05'), + array(array(2), '2014-04-06') + ); + + $this->assertEquals($expectedInvalidations, $api->getInvalidatedReports()); + } +} diff --git a/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php new file mode 100644 index 0000000000..fbf2c0b7a9 --- /dev/null +++ b/tests/PHPUnit/Integration/DataAccess/ArchiveInvalidatorTest.php @@ -0,0 +1,181 @@ +invalidator = new ArchiveInvalidator(); + } + + public function test_rememberToInvalidateArchivedReportsLater_shouldCreateAnEntryInCaseThereIsNoneYet() + { + $key = 'report_to_invalidate_2_2014-04-05'; + $this->assertFalse(Option::get($key)); + + $this->rememberReport(2, '2014-04-05'); + + $this->assertSame('1', Option::get($key)); + } + + public function test_rememberToInvalidateArchivedReportsLater_shouldNotCreateEntryTwice() + { + $this->rememberReport(2, '2014-04-05'); + $this->rememberReport(2, '2014-04-05'); + $this->rememberReport(2, '2014-04-05'); + + $this->assertCount(1, Option::getLike('report_to_invalidate%')); + } + + public function test_getRememberedArchivedReportsThatShouldBeInvalidated_shouldNotReturnEntriesInCaseNoneAreRemembered() + { + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + $this->assertSame(array(), $reports); + } + + public function test_getRememberedArchivedReportsThatShouldBeInvalidated_shouldGroupEntriesByDate() + { + $this->rememberReportsForManySitesAndDates(); + + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + $this->assertSame($this->getRememberedReportsByDate(), $reports); + } + + public function test_forgetRememberedArchivedReportsToInvalidateForSite_shouldNotDeleteAnythingInCaseNoReportForThatSite() + { + $this->rememberReportsForManySitesAndDates(); + + $this->invalidator->forgetRememberedArchivedReportsToInvalidateForSite(10); + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + $this->assertSame($this->getRememberedReportsByDate(), $reports); + } + + public function test_forgetRememberedArchivedReportsToInvalidateForSite_shouldOnlyDeleteReportsBelongingToThatSite() + { + $this->rememberReportsForManySitesAndDates(); + + $this->invalidator->forgetRememberedArchivedReportsToInvalidateForSite(7); + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + $expected = array( + '2014-04-05' => array(1, 2, 4), + '2014-05-05' => array(2, 5), + '2014-04-06' => array(3) + ); + $this->assertSame($expected, $reports); + } + + public function test_forgetRememberedArchivedReportsToInvalidate_shouldNotForgetAnythingIfThereIsNoMatch() + { + $this->rememberReportsForManySitesAndDates(); + + // site does not match + $this->invalidator->forgetRememberedArchivedReportsToInvalidate(10, Date::factory('2014-04-05')); + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + $this->assertSame($this->getRememberedReportsByDate(), $reports); + + // date does not match + $this->invalidator->forgetRememberedArchivedReportsToInvalidate(7, Date::factory('2012-04-05')); + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + $this->assertSame($this->getRememberedReportsByDate(), $reports); + } + + public function test_forgetRememberedArchivedReportsToInvalidate_shouldOnlyDeleteReportBelongingToThatSiteAndDate() + { + $this->rememberReportsForManySitesAndDates(); + + $this->invalidator->forgetRememberedArchivedReportsToInvalidate(2, Date::factory('2014-04-05')); + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + $expected = array( + '2014-04-05' => array(1, 4, 7), + '2014-05-05' => array(2, 5), + '2014-04-06' => array(3), + '2014-04-08' => array(7), + '2014-05-08' => array(7), + ); + $this->assertSame($expected, $reports); + + unset($expected['2014-05-08']); + + $this->invalidator->forgetRememberedArchivedReportsToInvalidate(7, Date::factory('2014-05-08')); + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + $this->assertSame($expected, $reports); + } + + public function test_markArchivesAsInvalidated_shouldForgetInvalidatedSitesAndDates() + { + $this->rememberReportsForManySitesAndDates(); + + $idSites = array(2, 10, 7, 5); + $dates = '2014-04-05,2014-04-08,2010-10-10'; + $this->invalidator->markArchivesAsInvalidated($idSites, $dates, false); + $reports = $this->invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + $expected = array( + '2014-04-05' => array(1, 4), + '2014-05-05' => array(2, 5), + '2014-04-06' => array(3), + '2014-05-08' => array(7), + ); + $this->assertSame($expected, $reports); + } + + private function rememberReport($idSite, $date) + { + $date = Date::factory($date); + $this->invalidator->rememberToInvalidateArchivedReportsLater($idSite, $date); + } + + private function getRememberedReportsByDate() + { + return array( + '2014-04-05' => array(1, 2, 4, 7), + '2014-05-05' => array(2, 5), + '2014-04-06' => array(3), + '2014-04-08' => array(7), + '2014-05-08' => array(7), + ); + } + + private function rememberReportsForManySitesAndDates() + { + $this->rememberReport(2, '2014-04-05'); + $this->rememberReport(2, '2014-04-05'); // should appear only once for this site and date + $this->rememberReport(3, '2014-04-06'); + $this->rememberReport(1, '2014-04-05'); + $this->rememberReport(2, '2014-05-05'); + $this->rememberReport(5, '2014-05-05'); + $this->rememberReport(4, '2014-04-05'); + $this->rememberReport(7, '2014-04-05'); + $this->rememberReport(7, '2014-05-08'); + $this->rememberReport(7, '2014-04-08'); + } +} diff --git a/tests/PHPUnit/Integration/Tracker/VisitTest.php b/tests/PHPUnit/Integration/Tracker/VisitTest.php index 41fcccb76e..604857f7c4 100644 --- a/tests/PHPUnit/Integration/Tracker/VisitTest.php +++ b/tests/PHPUnit/Integration/Tracker/VisitTest.php @@ -11,10 +11,12 @@ namespace Piwik\Tests\Integration\Tracker; use Piwik\Access; use Piwik\Cache; use Piwik\CacheId; +use Piwik\DataAccess\ArchiveInvalidator; use Piwik\Date; use Piwik\Network\IPUtils; use Piwik\Plugin\Manager; use Piwik\Plugins\SitesManager\API; +use Piwik\Tests\Framework\Fixture; use Piwik\Tests\Framework\Mock\FakeAccess; use Piwik\Tracker\ActionPageview; use Piwik\Tracker\Request; @@ -37,7 +39,8 @@ class VisitTest extends IntegrationTestCase FakeAccess::$superUser = true; Access::setSingletonInstance($pseudoMockAccess); - Manager::getInstance()->loadPlugins(array('SitesManager')); + Manager::getInstance()->loadTrackerPlugins(); + Manager::getInstance()->loadPlugin('SitesManager'); } /** @@ -280,6 +283,93 @@ class VisitTest extends IntegrationTestCase $this->assertFalse($result); } + public function test_markArchivedReportsAsInvalidIfArchiveAlreadyFinished_ShouldRemember_IfRequestWasDoneLongAgo() + { + $currentActionTime = '2012-01-02 08:12:45'; + $idsite = API::getInstance()->addSite('name', 'http://piwik.net/'); + + $expectedRemembered = array('2012-01-02' => array($idsite)); + + $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $currentActionTime, $expectedRemembered); + } + + public function test_markArchivedReportsAsInvalidIfArchiveAlreadyFinished_ShouldNotRemember_IfRequestWasDoneJustAtStartOfTheDay() + { + $currentActionTime = Date::today()->getDatetime(); + $idsite = API::getInstance()->addSite('name', 'http://piwik.net/'); + + $expectedRemembered = array(); + + $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $currentActionTime, $expectedRemembered); + } + + public function test_markArchivedReportsAsInvalidIfArchiveAlreadyFinished_ShouldRemember_IfRequestWasDoneAt11PMTheDayBefore() + { + $currentActionTime = Date::today()->subHour(1)->getDatetime(); + $idsite = API::getInstance()->addSite('name', 'http://piwik.net/'); + + $expectedRemembered = array( + substr($currentActionTime, 0, 10) => array($idsite) + ); + + $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $currentActionTime, $expectedRemembered); + } + + public function test_markArchivedReportsAsInvalidIfArchiveAlreadyFinished_shouldConsiderWebsitesTimezone() + { + $timezone1 = 'UTC+4'; + $timezone2 = 'UTC+6'; + + $currentActionTime1 = Date::today()->setTimezone($timezone1)->getDatetime(); + $currentActionTime2 = Date::today()->setTimezone($timezone2)->getDatetime(); + $idsite = API::getInstance()->addSite('name', 'http://piwik.net/', $ecommerce = null, + $siteSearch = null, + $searchKeywordParameters = null, + $searchCategoryParameters = null, + $excludedIps = null, + $excludedQueryParameters = null, + $timezone = 'UTC+5'); + + $expectedRemembered = array( + substr($currentActionTime1, 0, 10) => array($idsite) + ); + + // if website timezone was von considered both would be today (expected = array()) + $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $currentActionTime1, array()); + $this->assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $currentActionTime2, $expectedRemembered); + } + + private function assertRememberedArchivedReportsThatShouldBeInvalidated($idsite, $requestDate, $expectedRemeberedArchivedReports) + { + /** @var Visit $visit */ + list($visit) = $this->prepareVisitWithRequest(array( + 'idsite' => $idsite, + 'rec' => 1, + 'cip' => '156.146.156.146', + 'token_auth' => Fixture::getTokenAuth() + ), $requestDate); + + $visit->handle(); + + $archive = new ArchiveInvalidator(); + $remembered = $archive->getRememberedArchivedReportsThatShouldBeInvalidated(); + + $this->assertSame($expectedRemeberedArchivedReports, $remembered); + } + + private function prepareVisitWithRequest($requestParams, $requestDate) + { + $request = new Request($requestParams); + $request->setCurrentTimestamp(Date::factory($requestDate)->getTimestamp()); + + $visit = new Visit(); + $visit->setRequest($request); + + $visit->handle(); + + return array($visit, $request); + } + public function test_isVisitNew_ReturnsTrue_IfLastActionTimestampIsNotWithinVisitTimeLength_AndNoDimensionForcesVisit_AndVisitorNotKnown() { $this->setDimensionsWithOnNewVisit(array(false, false, false)); @@ -304,7 +394,7 @@ class VisitTest extends IntegrationTestCase $this->assertTrue($result); } - public function test_isVisitNew_ReturnsTrue_IfDimensionForcesVisit_AndVisitorKnown() + public function test_isVisitNew_ReturnsTrue_IfDimensionForcesVisit_AndVisitorKnown() { $this->setDimensionsWithOnNewVisit(array(false, false, true)); @@ -320,11 +410,7 @@ class VisitTest extends IntegrationTestCase { $idsite = API::getInstance()->addSite("name", "http://piwik.net/"); - $request = new Request(array('idsite' => $idsite)); - $request->setCurrentTimestamp(Date::factory($currentActionTime)->getTimestamp()); - - $visit = new Visit(); - $visit->setRequest($request); + list($visit, $request) = $this->prepareVisitWithRequest(array('idsite' => $idsite), $currentActionTime); $visitor = new Visitor($request, 'configid', array('visit_last_action_time' => Date::factory($lastActionTimestamp)->getTimestamp())); $visitor->setIsVisitorKnown($isVisitorKnown); -- cgit v1.2.3 From a1c74a9043812cd133d65a5a23a4a72ae0cf1928 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Sun, 21 Dec 2014 23:43:30 +0100 Subject: invalidate reports if needed when building an archive requested via web --- core/Archive.php | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/core/Archive.php b/core/Archive.php index b407465a45..76a619f0bb 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -10,8 +10,10 @@ namespace Piwik; use Piwik\Archive\Parameters; use Piwik\ArchiveProcessor\Rules; +use Piwik\DataAccess\ArchiveInvalidator; use Piwik\DataAccess\ArchiveSelector; use Piwik\Period\Factory as PeriodFactory; +use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeApi; /** * The **Archive** class is used to query cached analytics statistics @@ -160,6 +162,11 @@ class Archive */ private $params; + /** + * \Piwik\Cache\Cache + */ + private static $cache; + /** * @param Parameters $params * @param bool $forceIndexedBySite Whether to force index the result of a query by site ID. @@ -480,6 +487,32 @@ class Archive return $recordName . "_" . $id; } + private function invalidatedReportsIfNeeded() + { + if (is_null(self::$cache)) { + self::$cache = Cache::getTransientCache(); + } + + $id = 'Archive.RememberedReportsInvalidated'; + + if (self::$cache->contains($id)) { + return; + } + + self::$cache->save($id, 1); + + $invalidator = new ArchiveInvalidator(); + $sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); + + foreach ($sitesPerDays as $date => $siteIds) { + if (!empty($siteIds)) { + // an advanced version would only invalidate siteIds for $this->params->getIdSites() but would make + // everything way more complex eg the cache above and which siteIds we pass here... + CoreAdminHomeApi::getInstance()->invalidateArchivedReports($siteIds, $date); + } + } + } + /** * Queries archive tables for data and returns the result. * @param array|string $archiveNames @@ -489,6 +522,8 @@ class Archive */ private function get($archiveNames, $archiveDataType, $idSubtable = null) { + $this->invalidatedReportsIfNeeded(); + if (!is_array($archiveNames)) { $archiveNames = array($archiveNames); } -- cgit v1.2.3 From 4823afe878f87bafb2df55c70db3bcab49fe3fb6 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Sun, 21 Dec 2014 23:59:53 +0100 Subject: we do not want to use the API here since it needs admin access --- core/Archive.php | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/core/Archive.php b/core/Archive.php index 76a619f0bb..208d6dfdce 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -13,7 +13,6 @@ use Piwik\ArchiveProcessor\Rules; use Piwik\DataAccess\ArchiveInvalidator; use Piwik\DataAccess\ArchiveSelector; use Piwik\Period\Factory as PeriodFactory; -use Piwik\Plugins\CoreAdminHome\API as CoreAdminHomeApi; /** * The **Archive** class is used to query cached analytics statistics @@ -505,12 +504,21 @@ class Archive $sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); foreach ($sitesPerDays as $date => $siteIds) { - if (!empty($siteIds)) { + if (empty($siteIds)) { + continue; + } + + try { // an advanced version would only invalidate siteIds for $this->params->getIdSites() but would make // everything way more complex eg the cache above and which siteIds we pass here... - CoreAdminHomeApi::getInstance()->invalidateArchivedReports($siteIds, $date); + $invalidator->markArchivesAsInvalidated($siteIds, $date, false); + } catch (\Exception $e) { + Site::clearCache(); + throw $e; } } + + Site::clearCache(); } /** -- cgit v1.2.3 From 6c4266d301fe32f36174bba3a0c0cfdeb018ad9f Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 22 Dec 2014 00:11:42 +0100 Subject: accessing the sitesmanager api would fail in case one does not have admin access. --- core/DataAccess/ArchiveInvalidator.php | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/core/DataAccess/ArchiveInvalidator.php b/core/DataAccess/ArchiveInvalidator.php index 962f9d4386..8ff0516f11 100644 --- a/core/DataAccess/ArchiveInvalidator.php +++ b/core/DataAccess/ArchiveInvalidator.php @@ -15,6 +15,8 @@ use Piwik\Option; use Piwik\Plugins\PrivacyManager\PrivacyManager; use Piwik\Period; use Piwik\Period\Week; +use Piwik\Plugins\SitesManager\Model as SitesManagerModel; +use Piwik\Site; /** * Marks archives as Invalidated by setting the done flag to a special value (see Model->updateArchiveAsInvalidated) @@ -112,7 +114,7 @@ class ArchiveInvalidator { $datesToInvalidate = $this->getDatesToInvalidateFromString($dates); $minDate = $this->getMinimumDateToInvalidate($datesToInvalidate); - \Piwik\Plugins\SitesManager\API::getInstance()->updateSiteCreatedTime($idSites, $minDate); + $this->updateSiteCreatedTime($idSites, $minDate); $datesByMonth = $this->getDatesByYearMonth($datesToInvalidate); $this->markArchivesInvalidatedFor($idSites, $period, $datesByMonth); @@ -128,6 +130,15 @@ class ArchiveInvalidator { return $this->makeOutputLogs(); } + private function updateSiteCreatedTime($idSites, Date $minDate) + { + $idSites = Site::getIdSitesFromIdSitesString($idSites); + $minDateSql = $minDate->subDay(1)->getDatetime(); + + $model = new SitesManagerModel(); + $model->updateSiteCreatedTime($idSites, $minDateSql); + } + /** * @param $toInvalidate * @return bool|Date -- cgit v1.2.3 From 15dbbdfb9f5ffb2877abc30634f56820740129da Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 22 Dec 2014 00:50:57 +0100 Subject: added an integration test to make sure the archiver report invalidation works as expected --- core/Archive.php | 2 +- .../Fixtures/TwoSitesTwoVisitorsDifferentDays.php | 2 +- ...torsTwoWebsitesDifferentDaysConversionsTest.php | 72 ++++++++++++++++++++++ 3 files changed, 74 insertions(+), 2 deletions(-) diff --git a/core/Archive.php b/core/Archive.php index 208d6dfdce..dcc14b7ad0 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -162,7 +162,7 @@ class Archive private $params; /** - * \Piwik\Cache\Cache + * @var \Piwik\Cache\Cache */ private static $cache; diff --git a/tests/PHPUnit/Fixtures/TwoSitesTwoVisitorsDifferentDays.php b/tests/PHPUnit/Fixtures/TwoSitesTwoVisitorsDifferentDays.php index 0281ac532e..154dc70c2b 100644 --- a/tests/PHPUnit/Fixtures/TwoSitesTwoVisitorsDifferentDays.php +++ b/tests/PHPUnit/Fixtures/TwoSitesTwoVisitorsDifferentDays.php @@ -77,7 +77,7 @@ class TwoSitesTwoVisitorsDifferentDays extends Fixture $startDate = null, $excludedUserAgents = null, $keepURLFragments = 1); // KEEP_URL_FRAGMENT_YES Yes for idSite 2 } - private function trackVisits() + public function trackVisits() { $dateTime = $this->dateTime; $idSite = $this->idSite1; diff --git a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php index 88727c30a2..8542e53774 100755 --- a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php +++ b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php @@ -8,6 +8,9 @@ namespace Piwik\Tests\System; use Piwik\Archive; +use Piwik\Cache; +use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\Option; use Piwik\Plugins\Goals\Archiver; use Piwik\Segment; use Piwik\Tests\Framework\TestCase\SystemTestCase; @@ -25,6 +28,9 @@ require_once PIWIK_INCLUDE_PATH . '/plugins/Goals/Goals.php'; */ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase { + /** + * @var TwoSitesTwoVisitorsDifferentDays + */ public static $fixture = null; // initialized below class definition /** @@ -135,6 +141,72 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase ); } + // TODO: this test should be in an integration test for Piwik\Archive. setup code for getting metrics from different + // plugins is non-trivial, so not done now. + public function test_Archive_getNumeric_shouldInvalidateRememberedReportsOncePerRequestIfNeeded() + { + // Tests that getting a visits summary metric (nb_visits) & a Goal's metric (Goal_revenue) + // at the same time works. + $dateTimeRange = '2010-01-03,2010-01-06'; + $columns = array('nb_visits', 'Goal_nb_conversions', 'nb_actions'); + $idSite1 = self::$fixture->idSite1; + + $archive = Archive::build($idSite1, 'range', $dateTimeRange); + $result = $archive->getNumeric($columns); + $this->assertEquals( + array( + 'nb_visits' => 4, + 'Goal_nb_conversions' => 6, + 'nb_actions' => 13 + ), + $result + ); + + $cache = Cache::getTransientCache(); + $this->assertTrue($cache->contains('Archive.RememberedReportsInvalidated')); + + $invalidator = new ArchiveInvalidator(); + + self::$fixture->trackVisits(); + + // trackVisits should remember archived reports as invalid + $this->assertNotEmpty($invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); + + // although there were new tracked visists it doesn'T change as the report invalidation is cached and was + // already invalidated in previous Archive::get(); + $archive = Archive::build($idSite1, 'range', $dateTimeRange); + $result = $archive->getNumeric($columns); + $this->assertEquals( + array( + 'nb_visits' => 4, + 'Goal_nb_conversions' => 6, + 'nb_actions' => 13 // actions should remain the same as nothing was reports are still marked as already processed + ), + $result + ); + + // make sure the caching in archive::get() worked and they are still to be invalidated + $this->assertNotEmpty($invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); + + // now we force to actually invalidate archived reports again and then archive will be rebuilt + $cache->delete('Archive.RememberedReportsInvalidated'); + + $archive = Archive::build($idSite1, 'range', $dateTimeRange); + $result = $archive->getNumeric($columns); + + // archive::get() should have invalidated them now as we cleared the lock / cache before + $this->assertEmpty($invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); + + $this->assertEquals( + array( + 'nb_visits' => 4, + 'Goal_nb_conversions' => 6, + 'nb_actions' => 26 // now actions should be increased as the reports were invalidated + ), + $result + ); + } + public static function getOutputPrefix() { return 'TwoVisitors_twoWebsites_differentDays_Conversions'; -- cgit v1.2.3 From e71f461adfd03b40504243104944fe6b70c79980 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 22 Dec 2014 01:14:06 +0100 Subject: it took me a long time to figure out where something was printed in the middle of any test... --- .../System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php | 3 ++- tests/PHPUnit/System/VisitsInPastInvalidateOldReportsTest.php | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php index a6ba72de3d..f8d3ab4f15 100755 --- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php +++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php @@ -97,7 +97,8 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest $countBlobs = Db::get()->fetchOne($sql); if($expectedRows != $countBlobs) { - var_export(Db::get()->fetchAll("SELECT * FROM " . Common::prefixTable($table) . " ORDER BY name, idarchive ASC")); + $output = Db::get()->fetchAll("SELECT * FROM " . Common::prefixTable($table) . " ORDER BY name, idarchive ASC"); + var_export('This is debug output from ' . __CLASS__ . ' in case of an error: ' . $output); } $this->assertEquals($expectedRows, $countBlobs, "$table: %s"); } diff --git a/tests/PHPUnit/System/VisitsInPastInvalidateOldReportsTest.php b/tests/PHPUnit/System/VisitsInPastInvalidateOldReportsTest.php index 15149f0ec5..859eda1646 100644 --- a/tests/PHPUnit/System/VisitsInPastInvalidateOldReportsTest.php +++ b/tests/PHPUnit/System/VisitsInPastInvalidateOldReportsTest.php @@ -22,6 +22,9 @@ use Exception; */ class VisitsInPastInvalidateOldReportsTest extends SystemTestCase { + /** + * @var TwoSitesVisitsInPast + */ public static $fixture = null; // initialized below class definition /** -- cgit v1.2.3 From 55fe05468d86dc51ad653b5de9bb4b9ad027ba78 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 22 Dec 2014 03:32:06 +0100 Subject: fix tests as reports are invalidated now in some tests causing the site creation date to change causing archiving more dates --- tests/PHPUnit/Fixtures/TwoSitesVisitsInPast.php | 22 ++++---- ...hCustomVariablesSegmentMatchVisitorTypeTest.php | 9 ++-- .../VisitsInPastInvalidateOldReportsTest.php | 8 +-- ...sShouldNotAppear__Actions.getPageUrls_month.xml | 59 +++++++++++++++++++++- ...rtsShouldNotAppear__VisitsSummary.get_month.xml | 14 ++++- ...sShouldNotAppear__Actions.getPageUrls_month.xml | 59 +++++++++++++++++++++- ...rtsShouldNotAppear__VisitsSummary.get_month.xml | 14 ++++- 7 files changed, 163 insertions(+), 22 deletions(-) diff --git a/tests/PHPUnit/Fixtures/TwoSitesVisitsInPast.php b/tests/PHPUnit/Fixtures/TwoSitesVisitsInPast.php index 5e508f9d0b..3ec434d552 100644 --- a/tests/PHPUnit/Fixtures/TwoSitesVisitsInPast.php +++ b/tests/PHPUnit/Fixtures/TwoSitesVisitsInPast.php @@ -15,10 +15,10 @@ use Piwik\Tests\Framework\Fixture; */ class TwoSitesVisitsInPast extends Fixture { - public $dateTimeFirstDateWebsite1 = '2010-03-06 01:22:33'; - public $dateTimeDateInPastWebsite1 = '2010-01-06 01:22:33'; - public $dateTimeFirstDateWebsite2 = '2010-01-03 20:22:33'; - public $dateTimeDateInPastWebsite2 = '2009-10-30 01:22:33'; + public $dateTimeCreationWebsite1 = '2010-03-06 01:22:33'; + public $dateTimeInPastWebsite1 = '2010-01-06 01:22:33'; + public $dateTimeCreationWebsite2 = '2010-01-03 20:22:33'; + public $dateTimeInPastWebsite2 = '2009-10-30 01:22:33'; public $idSite = 1; public $idSite2 = 2; @@ -36,11 +36,11 @@ class TwoSitesVisitsInPast extends Fixture public function setUpWebsitesAndGoals() { if (!self::siteCreated($idSite = 1)) { - self::createWebsite($this->dateTimeFirstDateWebsite1); + self::createWebsite($this->dateTimeCreationWebsite1); } if (!self::siteCreated($idSite = 2)) { - self::createWebsite($this->dateTimeFirstDateWebsite2); + self::createWebsite($this->dateTimeCreationWebsite2); } } @@ -50,7 +50,7 @@ class TwoSitesVisitsInPast extends Fixture * Track Visits normal date for the 2 websites */ // WEBSITE 1 - $t = self::getTracker($this->idSite, $this->dateTimeFirstDateWebsite1, $defaultInit = true); + $t = self::getTracker($this->idSite, $this->dateTimeCreationWebsite1, $defaultInit = true); $t->setUrl('http://example.org/category/Page1'); self::checkResponse($t->doTrackPageView('Hello')); $t->setUrl('http://example.org/category/Page2'); @@ -65,7 +65,7 @@ class TwoSitesVisitsInPast extends Fixture self::checkResponse($t->doTrackPageView('Hello')); // WEBSITE 2 - $t = self::getTracker($this->idSite2, $this->dateTimeFirstDateWebsite2, $defaultInit = true); + $t = self::getTracker($this->idSite2, $this->dateTimeCreationWebsite2, $defaultInit = true); $t->setIp('156.15.13.12'); $t->setUrl('http://example.org/category/Page1'); self::checkResponse($t->doTrackPageView('Hello')); @@ -84,7 +84,7 @@ class TwoSitesVisitsInPast extends Fixture * Track visits in the past (before website creation date) for the 2 websites */ // WEBSITE1 - $t = self::getTracker($this->idSite, $this->dateTimeDateInPastWebsite1, $defaultInit = true); + $t = self::getTracker($this->idSite, $this->dateTimeInPastWebsite1, $defaultInit = true); $t->setIp('156.5.55.2'); $t->setUrl('http://example.org/category/Page1'); self::checkResponse($t->doTrackPageView('Hello')); @@ -96,7 +96,7 @@ class TwoSitesVisitsInPast extends Fixture self::checkResponse($t->doTrackPageView('Blabla')); // WEBSITE2 - $t = self::getTracker($this->idSite2, $this->dateTimeDateInPastWebsite2, $defaultInit = true); + $t = self::getTracker($this->idSite2, $this->dateTimeInPastWebsite2, $defaultInit = true); $t->setIp('156.52.3.22'); $t->setUrl('http://example.org/category/Page1'); self::checkResponse($t->doTrackPageView('Hello')); @@ -106,7 +106,7 @@ class TwoSitesVisitsInPast extends Fixture self::checkResponse($t->doTrackPageView('Hello')); $t->setUrl('http://example.org/category/Pageyy'); self::checkResponse($t->doTrackPageView('Blabla')); - $t->setForceVisitDateTime(Date::factory($this->dateTimeDateInPastWebsite2)->addHour(0.1)->getDatetime()); + $t->setForceVisitDateTime(Date::factory($this->dateTimeInPastWebsite2)->addHour(0.1)->getDatetime()); $t->setUrl('http://example.org/category/Pageyy'); self::checkResponse($t->doTrackPageView('Blabla')); } diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php index f8d3ab4f15..06ee848eed 100755 --- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php +++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php @@ -8,6 +8,8 @@ namespace Piwik\Tests\System; use Piwik\Common; +use Piwik\DataAccess\ArchiveInvalidator; +use Piwik\DataAccess\InvalidatedReports; use Piwik\Db; use Piwik\Tests\Framework\TestCase\SystemTestCase; use Piwik\Tests\Fixtures\TwoVisitsWithCustomVariables; @@ -88,9 +90,9 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest 'archive_blob_2009_12' => 28, // 7 metrics, // 2 Referrer metrics (Referrers_distinctSearchEngines/Referrers_distinctKeywords), - // 3 done flag (referrers, CustomVar, VisitsSummary), + // 6 done flag (referrers, CustomVar, VisitsSummary), 3 for period = 1 and 3 for period = 2 // X * 2 segments - 'archive_numeric_2009_12' => (6 + 2 + 3) * 2, + 'archive_numeric_2009_12' => (6 + 2 + 3 + 3) * 2, ); foreach ($tests as $table => $expectedRows) { $sql = "SELECT count(*) FROM " . Common::prefixTable($table); @@ -98,7 +100,8 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest if($expectedRows != $countBlobs) { $output = Db::get()->fetchAll("SELECT * FROM " . Common::prefixTable($table) . " ORDER BY name, idarchive ASC"); - var_export('This is debug output from ' . __CLASS__ . ' in case of an error: ' . $output); + var_export('This is debug output from ' . __CLASS__ . ' in case of an error: '); + var_export($output); } $this->assertEquals($expectedRows, $countBlobs, "$table: %s"); } diff --git a/tests/PHPUnit/System/VisitsInPastInvalidateOldReportsTest.php b/tests/PHPUnit/System/VisitsInPastInvalidateOldReportsTest.php index 859eda1646..b951995368 100644 --- a/tests/PHPUnit/System/VisitsInPastInvalidateOldReportsTest.php +++ b/tests/PHPUnit/System/VisitsInPastInvalidateOldReportsTest.php @@ -42,8 +42,8 @@ class VisitsInPastInvalidateOldReportsTest extends SystemTestCase { $idSite = self::$fixture->idSite; $idSite2 = self::$fixture->idSite2; - $dateTimeDateInPastWebsite1 = self::$fixture->dateTimeDateInPastWebsite1; - $dateTimeDateInPastWebsite2 = self::$fixture->dateTimeDateInPastWebsite2; + $dateTimeDateInPastWebsite1 = self::$fixture->dateTimeInPastWebsite1; + $dateTimeDateInPastWebsite2 = self::$fixture->dateTimeInPastWebsite2; // We test a typical Numeric and a Recursive blob reports $apiToCall = array('VisitsSummary.get', 'Actions.getPageUrls'); @@ -124,8 +124,8 @@ class VisitsInPastInvalidateOldReportsTest extends SystemTestCase { $idSite = self::$fixture->idSite; $idSite2 = self::$fixture->idSite2; - $dateTimeDateInPastWebsite1 = self::$fixture->dateTimeDateInPastWebsite1; - $dateTimeDateInPastWebsite2 = self::$fixture->dateTimeDateInPastWebsite2; + $dateTimeDateInPastWebsite1 = self::$fixture->dateTimeInPastWebsite1; + $dateTimeDateInPastWebsite2 = self::$fixture->dateTimeInPastWebsite2; $apiToCall = array('VisitsSummary.get', 'Actions.getPageUrls'); diff --git a/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite1_OldReportsShouldNotAppear__Actions.getPageUrls_month.xml b/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite1_OldReportsShouldNotAppear__Actions.getPageUrls_month.xml index 89eec3fbc3..2f10eb3f97 100644 --- a/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite1_OldReportsShouldNotAppear__Actions.getPageUrls_month.xml +++ b/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite1_OldReportsShouldNotAppear__Actions.getPageUrls_month.xml @@ -1,6 +1,63 @@ - + + + + 3 + 4 + 0 + 1 + 4 + 1 + 0 + 1 + 0 + 0% + 33% + + + + 1 + 2 + 0 + 1 + 4 + 1 + 0 + 1 + 1 + 0 + 0% + 0% + http://example.org/category/Page1 + + + + 1 + 1 + 0 + 1 + 0 + 0% + 0% + http://example.org/category/Page2 + + + + 1 + 1 + 0 + 1 + 1 + 1 + 0 + 0% + 100% + http://example.org/category/Pagexx + + + + diff --git a/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite1_OldReportsShouldNotAppear__VisitsSummary.get_month.xml b/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite1_OldReportsShouldNotAppear__VisitsSummary.get_month.xml index d84e6c4f35..2f7ce19aa5 100644 --- a/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite1_OldReportsShouldNotAppear__VisitsSummary.get_month.xml +++ b/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite1_OldReportsShouldNotAppear__VisitsSummary.get_month.xml @@ -1,6 +1,18 @@ - + + 1 + 0 + 1 + 4 + 0 + 0 + 1 + 4 + 0% + 4 + 1 + 1 diff --git a/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite2_OldReportsShouldNotAppear__Actions.getPageUrls_month.xml b/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite2_OldReportsShouldNotAppear__Actions.getPageUrls_month.xml index aaef3a0ec5..401e8d2dc7 100644 --- a/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite2_OldReportsShouldNotAppear__Actions.getPageUrls_month.xml +++ b/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite2_OldReportsShouldNotAppear__Actions.getPageUrls_month.xml @@ -1,6 +1,63 @@ - + + + + 3 + 5 + 360 + 1 + 5 + 361 + 0 + 1 + 120 + 0% + 33% + + + + 1 + 2 + 0 + 1 + 5 + 361 + 0 + 1 + 1 + 0 + 0% + 0% + http://example.org/category/Page1 + + + + 1 + 1 + 0 + 1 + 0 + 0% + 0% + http://example.org/category/Page2 + + + + 1 + 2 + 360 + 1 + 1 + 1 + 360 + 0% + 100% + http://example.org/category/Pageyy + + + + diff --git a/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite2_OldReportsShouldNotAppear__VisitsSummary.get_month.xml b/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite2_OldReportsShouldNotAppear__VisitsSummary.get_month.xml index d046dca549..ed3e2b6ade 100644 --- a/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite2_OldReportsShouldNotAppear__VisitsSummary.get_month.xml +++ b/tests/PHPUnit/System/expected/test_VisitsInPast_InvalidateOldReportsWebsite2_OldReportsShouldNotAppear__VisitsSummary.get_month.xml @@ -1,6 +1,18 @@ - + + 1 + 0 + 1 + 5 + 0 + 0 + 361 + 5 + 0% + 5 + 361 + -- cgit v1.2.3 From 4e1563c553e8db798aa3f615abb6106336339eca Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 22 Dec 2014 04:35:19 +0100 Subject: only invalidate reports for requested site ids for better performance --- core/Archive.php | 44 ++++++++++++++++++---- ...torsTwoWebsitesDifferentDaysConversionsTest.php | 16 ++++---- 2 files changed, 45 insertions(+), 15 deletions(-) diff --git a/core/Archive.php b/core/Archive.php index dcc14b7ad0..eada3e2b53 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -486,19 +486,43 @@ class Archive return $recordName . "_" . $id; } - private function invalidatedReportsIfNeeded() + private function getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet() { if (is_null(self::$cache)) { self::$cache = Cache::getTransientCache(); } - $id = 'Archive.RememberedReportsInvalidated'; + $id = 'Archive.SiteIdsOfRememberedReportsInvalidated'; - if (self::$cache->contains($id)) { - return; + if (!self::$cache->contains($id)) { + self::$cache->save($id, array()); } - self::$cache->save($id, 1); + $siteIdsAlreadyHandled = self::$cache->fetch($id); + $siteIdsRequested = $this->params->getIdSites(); + + foreach ($siteIdsRequested as $index => $siteIdRequested) { + $siteIdRequested = (int) $siteIdRequested; + + if (in_array($siteIdRequested, $siteIdsAlreadyHandled)) { + unset($siteIdsRequested[$index]); // was already handled previously, do not do it again + } else { + $siteIdsAlreadyHandled[] = $siteIdRequested; // we will handle this id this time + } + } + + self::$cache->save($id, $siteIdsAlreadyHandled); + + return $siteIdsRequested; + } + + private function invalidatedReportsIfNeeded() + { + $siteIdsRequested = $this->getSiteIdsThatAreRequestedInThisArchiveButWereNotInvalidatedYet(); + + if (empty($siteIdsRequested)) { + return; // all requested site ids were already handled + } $invalidator = new ArchiveInvalidator(); $sitesPerDays = $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated(); @@ -508,10 +532,14 @@ class Archive continue; } + $siteIdsToActuallyInvalidate = array_intersect($siteIds, $siteIdsRequested); + + if (empty($siteIdsToActuallyInvalidate)) { + continue; // all site ids that should be handled are already handled + } + try { - // an advanced version would only invalidate siteIds for $this->params->getIdSites() but would make - // everything way more complex eg the cache above and which siteIds we pass here... - $invalidator->markArchivesAsInvalidated($siteIds, $date, false); + $invalidator->markArchivesAsInvalidated($siteIdsToActuallyInvalidate, $date, false); } catch (\Exception $e) { Site::clearCache(); throw $e; diff --git a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php index 8542e53774..712608c62e 100755 --- a/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php +++ b/tests/PHPUnit/System/TwoVisitorsTwoWebsitesDifferentDaysConversionsTest.php @@ -163,13 +163,14 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase ); $cache = Cache::getTransientCache(); - $this->assertTrue($cache->contains('Archive.RememberedReportsInvalidated')); + $this->assertEquals(array(self::$fixture->idSite1, self::$fixture->idSite2), + $cache->fetch('Archive.SiteIdsOfRememberedReportsInvalidated')); $invalidator = new ArchiveInvalidator(); self::$fixture->trackVisits(); - // trackVisits should remember archived reports as invalid + // trackVisits should remember to invalidate archived reports $this->assertNotEmpty($invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); // although there were new tracked visists it doesn'T change as the report invalidation is cached and was @@ -186,16 +187,17 @@ class TwoVisitorsTwoWebsitesDifferentDaysConversionsTest extends SystemTestCase ); // make sure the caching in archive::get() worked and they are still to be invalidated - $this->assertNotEmpty($invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); + $this->assertCount(10, $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); - // now we force to actually invalidate archived reports again and then archive will be rebuilt - $cache->delete('Archive.RememberedReportsInvalidated'); + // now we force to actually invalidate archived reports again and then archive will be rebuilt for requsted siteId = 1 + $cache->delete('Archive.SiteIdsOfRememberedReportsInvalidated'); $archive = Archive::build($idSite1, 'range', $dateTimeRange); $result = $archive->getNumeric($columns); - // archive::get() should have invalidated them now as we cleared the lock / cache before - $this->assertEmpty($invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); + // archive::get() should have invalidated siteId 1 and siteId 2 should be still to be done + $expectedArchiveReportsLeft = array('2010-01-04' => array(2)); + $this->assertEquals($expectedArchiveReportsLeft, $invalidator->getRememberedArchivedReportsThatShouldBeInvalidated()); $this->assertEquals( array( -- cgit v1.2.3 From 9782368a93e958ce32f504acc153495992ccc1b2 Mon Sep 17 00:00:00 2001 From: Thomas Steur Date: Mon, 22 Dec 2014 05:12:57 +0100 Subject: this should fix the remaining test as we clear the cache for all sites after invalidating an archived report all previously created site instances lost their data and need to be repopulated --- core/Site.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/core/Site.php b/core/Site.php index 6ac0c33f57..1a828fce47 100644 --- a/core/Site.php +++ b/core/Site.php @@ -237,7 +237,13 @@ class Site protected function get($name) { if (!isset(self::$infoSites[$this->id])) { - throw new UnexpectedWebsiteFoundException('The requested website id = ' . (int)$this->id . ' couldn\'t be found'); + $site = API::getInstance()->getSiteFromId($this->id); + + if (empty($site)) { + throw new UnexpectedWebsiteFoundException('The requested website id = ' . (int)$this->id . ' couldn\'t be found'); + } + + self::setSite($this->id, $site); } if (!isset(self::$infoSites[$this->id][$name])) { throw new Exception("The property $name could not be found on the website ID " . (int)$this->id); -- cgit v1.2.3