From eac5a58130665b5689ef7d2bb4ddc4367368b47f Mon Sep 17 00:00:00 2001 From: alexkuk Date: Tue, 19 Apr 2016 05:26:51 +0300 Subject: Implements #9389 - UserId module, that adds a new Users report (#9883) * Implement the UserId module, that adds a new Users report showing all unique user IDs and some aggregated data. Includes reindexing of raw visitors log into aggregated user ids table * Update UserId module: add total_actions, total_events and total_searches columns; format dates * Use an injected model instead of creating a new object * Rework the UserId plugin to use core archiver instead of custom indexer * Users report small changes: - remove logger injection - change archive record name - add low population filter - add all columns visualization - add datatable_archiving_maximum_rows_userid_users configuration, default value is 50000 * Remove unused method parameter * Users report: remove custom visualizations and add data-row-metadata attribute for every row --- plugins/UserId/.gitignore | 1 + plugins/UserId/API.php | 49 +++++++ plugins/UserId/Archiver.php | 139 +++++++++++++++++++ plugins/UserId/Reports/Base.php | 19 +++ plugins/UserId/Reports/GetUsers.php | 67 +++++++++ plugins/UserId/UserId.php | 53 +++++++ plugins/UserId/images/visitordetails-hover.png | Bin 0 -> 1275 bytes plugins/UserId/images/visitordetails.png | Bin 0 -> 1273 bytes plugins/UserId/javascripts/rowaction.js | 73 ++++++++++ .../Fixtures/TrackFewVisitsAndCreateUsers.php | 58 ++++++++ plugins/UserId/tests/System/ApiTest.php | 154 +++++++++++++++++++++ .../System/expected/test___UserId.getUsers_day.xml | 39 ++++++ .../expected/test___UserId.getUsers_range.xml | 39 ++++++ .../test_ascSortOrder__UserId.getUsers_day.xml | 39 ++++++ .../test_ascSortOrder__UserId.getUsers_range.xml | 39 ++++++ .../expected/test_limit__UserId.getUsers_day.xml | 27 ++++ .../expected/test_limit__UserId.getUsers_range.xml | 27 ++++ .../test_searchByUserId__UserId.getUsers_day.xml | 15 ++ .../test_searchByUserId__UserId.getUsers_range.xml | 15 ++ 19 files changed, 853 insertions(+) create mode 100644 plugins/UserId/.gitignore create mode 100644 plugins/UserId/API.php create mode 100644 plugins/UserId/Archiver.php create mode 100644 plugins/UserId/Reports/Base.php create mode 100644 plugins/UserId/Reports/GetUsers.php create mode 100644 plugins/UserId/UserId.php create mode 100644 plugins/UserId/images/visitordetails-hover.png create mode 100644 plugins/UserId/images/visitordetails.png create mode 100644 plugins/UserId/javascripts/rowaction.js create mode 100644 plugins/UserId/tests/Fixtures/TrackFewVisitsAndCreateUsers.php create mode 100644 plugins/UserId/tests/System/ApiTest.php create mode 100644 plugins/UserId/tests/System/expected/test___UserId.getUsers_day.xml create mode 100644 plugins/UserId/tests/System/expected/test___UserId.getUsers_range.xml create mode 100644 plugins/UserId/tests/System/expected/test_ascSortOrder__UserId.getUsers_day.xml create mode 100644 plugins/UserId/tests/System/expected/test_ascSortOrder__UserId.getUsers_range.xml create mode 100644 plugins/UserId/tests/System/expected/test_limit__UserId.getUsers_day.xml create mode 100644 plugins/UserId/tests/System/expected/test_limit__UserId.getUsers_range.xml create mode 100644 plugins/UserId/tests/System/expected/test_searchByUserId__UserId.getUsers_day.xml create mode 100644 plugins/UserId/tests/System/expected/test_searchByUserId__UserId.getUsers_range.xml (limited to 'plugins/UserId') diff --git a/plugins/UserId/.gitignore b/plugins/UserId/.gitignore new file mode 100644 index 0000000000..c8c9480010 --- /dev/null +++ b/plugins/UserId/.gitignore @@ -0,0 +1 @@ +tests/System/processed/*xml \ No newline at end of file diff --git a/plugins/UserId/API.php b/plugins/UserId/API.php new file mode 100644 index 0000000000..6e29c3afc9 --- /dev/null +++ b/plugins/UserId/API.php @@ -0,0 +1,49 @@ +getDataTable(Archiver::USERID_ARCHIVE_RECORD); + + $dataTable->queueFilter('ReplaceColumnNames'); + $dataTable->queueFilter('ReplaceSummaryRowLabel'); + + return $dataTable; + } +} diff --git a/plugins/UserId/Archiver.php b/plugins/UserId/Archiver.php new file mode 100644 index 0000000000..3e20dcf141 --- /dev/null +++ b/plugins/UserId/Archiver.php @@ -0,0 +1,139 @@ +maximumRowsInDataTableLevelZero = Config::getInstance()->General['datatable_archiving_maximum_rows_userid_users']; + } + + /** + * @var DataArray + */ + protected $arrays; + + /** + * Array to save visitor IDs for every user ID met during archiving process. We use it to + * fill metadata before actual inserting rows to DB. + * @var array + */ + protected $visitorIdsUserIdsMap = array(); + + /** + * Archives data for a day period. + */ + public function aggregateDayReport() + { + $this->arrays = new DataArray(); + $this->aggregateUsers(); + $this->insertDayReports(); + } + /** + * Period archiving: simply sums up daily archives + */ + public function aggregateMultipleReports() + { + $dataTableRecords = array(self::USERID_ARCHIVE_RECORD); + $columnsAggregationOperation = null; + $this->getProcessor()->aggregateDataTableRecords( + $dataTableRecords, + $this->maximumRowsInDataTableLevelZero, + $this->maximumRowsInDataTableLevelZero, + $columnToSort = 'nb_visits', + $columnsAggregationOperation, + $columnsToRenameAfterAggregation = null, + $countRowsRecursive = array()); + } + + /** + * Used to aggregate daily data per user ID + */ + protected function aggregateUsers() + { + $userIdFieldName = self::USER_ID_FIELD; + $visitorIdFieldName = self::VISITOR_ID_FIELD; + + /** @var Zend_Db_Statement $query */ + $query = $this->getLogAggregator()->queryVisitsByDimension( + array(self::USER_ID_FIELD), + "$userIdFieldName IS NOT NULL AND $userIdFieldName != ''", + array("LOWER(HEX($visitorIdFieldName)) as $visitorIdFieldName") + ); + if ($query === false) { + return; + } + + $rowsCount = 0; + while ($row = $query->fetch()) { + $rowsCount++; + $this->arrays->sumMetricsVisits($row[$userIdFieldName], $row); + $this->rememberVisitorId($row); + } + } + + /** + * Insert aggregated daily data serialized + * + * @throws \Exception + */ + protected function insertDayReports() + { + /** @var DataTable $dataTable */ + $dataTable = $this->arrays->asDataTable(); + $this->setVisitorIds($dataTable); + $report = $dataTable->getSerialized($this->maximumRowsInDataTableLevelZero, null, Metrics::INDEX_NB_VISITS); + $this->getProcessor()->insertBlobRecord(self::USERID_ARCHIVE_RECORD, $report); + } + + /** + * Remember visitor ID per user. We use it to fill metadata before actual inserting rows to DB. + * + * @param array $row + */ + protected function rememberVisitorId($row) + { + if (!empty($row[self::USER_ID_FIELD]) && !empty($row[self::VISITOR_ID_FIELD])) { + $this->visitorIdsUserIdsMap[$row[self::USER_ID_FIELD]] = $row[self::VISITOR_ID_FIELD]; + } + } + + /** + * Fill visitor ID as metadata before actual inserting rows to DB. + * + * @param DataTable $dataTable + */ + protected function setVisitorIds(DataTable $dataTable) + { + foreach ($dataTable->getRows() as $row) { + $userId = $row->getColumn('label'); + if (isset($this->visitorIdsUserIdsMap[$userId])) { + $row->setMetadata(self::VISITOR_ID_FIELD, $this->visitorIdsUserIdsMap[$userId]); + } + } + } + +} \ No newline at end of file diff --git a/plugins/UserId/Reports/Base.php b/plugins/UserId/Reports/Base.php new file mode 100644 index 0000000000..1971a19840 --- /dev/null +++ b/plugins/UserId/Reports/Base.php @@ -0,0 +1,19 @@ +category = 'General_Visitors'; + } +} diff --git a/plugins/UserId/Reports/GetUsers.php b/plugins/UserId/Reports/GetUsers.php new file mode 100644 index 0000000000..c5914776c5 --- /dev/null +++ b/plugins/UserId/Reports/GetUsers.php @@ -0,0 +1,67 @@ +name = Piwik::translate('UsersManager_MenuUsers'); + $this->menuTitle = $this->name; + $this->documentation = ''; + + // This defines in which order your report appears in the mobile app, in the menu and in the list of widgets + $this->order = 1; + } + + /** + * @param ViewDataTable $view + */ + public function configureView(ViewDataTable $view) + { + $view->config->addTranslation('label', Piwik::translate('General_UserId')); + $view->config->addTranslation('nb_visits_converted', Piwik::translate('General_VisitConvertedGoal')); + + /* + * Hide most of the table footer actions, leaving only export icons and pagination + */ + $view->config->columns_to_display = $this->getColumnsToDisplay(); + $view->config->show_all_views_icons = false; + $view->config->show_active_view_icon = false; + $view->config->show_related_reports = false; + $view->config->show_insights = false; + $view->config->show_pivot_by_subtable = false; + $view->config->show_flatten_table = false; + $view->config->disable_row_evolution = true; + + // exclude users with less then 2 visits, when low population filter is active + $view->requestConfig->filter_excludelowpop_value = 2; + } +} diff --git a/plugins/UserId/UserId.php b/plugins/UserId/UserId.php new file mode 100644 index 0000000000..b69f47c4e7 --- /dev/null +++ b/plugins/UserId/UserId.php @@ -0,0 +1,53 @@ + 'getJavaScriptFiles', + // Add translations for the client side JS + 'Translate.getClientSideTranslationKeys' => 'getClientSideTranslationKeys', + ); + } + + /** + * Add a custom JS to the page. It adds possibility to open visitor details popover for each + * user ID in a report table + * + * @param $jsFiles + */ + public function getJavaScriptFiles(&$jsFiles) + { + $jsFiles[] = "plugins/UserId/javascripts/rowaction.js"; + } + + /** + * Add translations for the client side JS + * + * @param $translationKeys + */ + public function getClientSideTranslationKeys(&$translationKeys) + { + $translationKeys[] = "Live_ViewVisitorProfile"; + } +} diff --git a/plugins/UserId/images/visitordetails-hover.png b/plugins/UserId/images/visitordetails-hover.png new file mode 100644 index 0000000000..523a8ac6b4 Binary files /dev/null and b/plugins/UserId/images/visitordetails-hover.png differ diff --git a/plugins/UserId/images/visitordetails.png b/plugins/UserId/images/visitordetails.png new file mode 100644 index 0000000000..423392db75 Binary files /dev/null and b/plugins/UserId/images/visitordetails.png differ diff --git a/plugins/UserId/javascripts/rowaction.js b/plugins/UserId/javascripts/rowaction.js new file mode 100644 index 0000000000..5fba72dbe4 --- /dev/null +++ b/plugins/UserId/javascripts/rowaction.js @@ -0,0 +1,73 @@ +/*! + * Piwik - free/libre analytics platform + * + * @link http://piwik.org + * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later + */ + +/** + * This file registers the visitor details overlay row action on the user IDs list page. + */ +(function () { + + var actionName = 'visitorDetails'; + + function DataTable_RowActions_VisitorDetails(dataTable) { + this.dataTable = dataTable; + this.actionName = actionName; + this.trEventName = 'piwikTriggerVisitorDetailsAction'; + } + + DataTable_RowActions_VisitorDetails.prototype = new DataTable_RowAction(); + + DataTable_RowActions_VisitorDetails.prototype.performAction = function (label, tr, e) { + var visitorId = this.getRowMetadata($(tr)).idvisitor || ''; + if (visitorId.length > 0) { + DataTable_RowAction.prototype.openPopover.apply(this, ['module=Live&action=getVisitorProfilePopup&visitorId=' + visitorId]); + } + }; + + DataTable_RowActions_VisitorDetails.prototype.doOpenPopover = function (urlParam) { + Piwik_Popover.createPopupAndLoadUrl(urlParam, _pk_translate('Live_VisitorProfile'), 'visitor-profile-popup'); + }; + + DataTable_RowActions_Registry.register({ + + name: actionName, + + instance: null, + + dataTableIcon: 'plugins/UserId/images/visitordetails.png', + dataTableIconHover: 'plugins/UserId/images/visitordetails-hover.png', + + order: 30, + + dataTableIconTooltip: [ + _pk_translate('Live_ViewVisitorProfile'), + '' + ], + + isAvailableOnReport: function (dataTableParams, undefined) { + return dataTableParams.module == 'UserId'; + }, + + isAvailableOnRow: function (dataTableParams, tr) { + return DataTable_RowAction.prototype.getRowMetadata(tr).hasOwnProperty('idvisitor'); + }, + + createInstance: function (dataTable, param) { + if (dataTable !== null && typeof dataTable.visitorDetailsInstance != 'undefined') { + return dataTable.segmentVisitorLogInstance; + } + + var instance = new DataTable_RowActions_VisitorDetails(dataTable); + if (dataTable !== null) { + dataTable.visitorDetailsInstance = instance; + } + + this.instance = instance; + + return instance; + } + }); +})(); \ No newline at end of file diff --git a/plugins/UserId/tests/Fixtures/TrackFewVisitsAndCreateUsers.php b/plugins/UserId/tests/Fixtures/TrackFewVisitsAndCreateUsers.php new file mode 100644 index 0000000000..3c5ac0fcbd --- /dev/null +++ b/plugins/UserId/tests/Fixtures/TrackFewVisitsAndCreateUsers.php @@ -0,0 +1,58 @@ +dateTime); + } + + $this->trackVisits(); + } + + private function trackVisits() + { + $t = self::getTracker($this->idSite, $this->dateTime, $defaultInit = true); + $t->setTokenAuth(self::getTokenAuth()); + $t->enableBulkTracking(); + + foreach (array('user1', 'user2', 'user3') as $key => $userId) { + for ($numVisits = 0; $numVisits < ($key+1) * 10; $numVisits++) { + $t->setUserId($userId); + if ($numVisits % 5 == 0) { + $t->doTrackSiteSearch('some search term'); + } + if ($numVisits % 4 == 0) { + $t->doTrackEvent('Event action', 'event cat'); + } + $t->setForceNewVisit(); + $t->setUrl('http://example.org/my/dir/page' . ($numVisits % 4)); + + $visitDateTime = Date::factory($this->dateTime)->addDay($numVisits)->getDatetime(); + $t->setForceVisitDateTime($visitDateTime); + + self::assertTrue($t->doTrackPageView('incredible title ' . ($numVisits % 3))); + } + } + + self::checkBulkTrackingResponse($t->doBulkTrack()); + } +} \ No newline at end of file diff --git a/plugins/UserId/tests/System/ApiTest.php b/plugins/UserId/tests/System/ApiTest.php new file mode 100644 index 0000000000..ca73c2792d --- /dev/null +++ b/plugins/UserId/tests/System/ApiTest.php @@ -0,0 +1,154 @@ +runApiTests($api, $params); + } + + public function getApiForTesting() + { + $api = 'UserId.getUsers'; + $startDate = substr(self::$fixture->dateTime, 0, 10); + $endDate = date('Y-m-d', strtotime($startDate) + 3600*24*365); + + $apiToTest = array(); + $apiToTest[] = array( + $api, + array( + 'date' => $startDate, + 'periods' => array('day'), + 'idSite' => 1, + 'testSuffix' => '' + ) + ); + $apiToTest[] = array( + $api, + array( + 'date' => "$startDate,$endDate", + 'periods' => array('range'), + 'idSite' => 1, + 'testSuffix' => '' + ) + ); + + $apiToTest[] = array( + $api, + array( + 'date' => $startDate, + 'periods' => array('day'), + 'idSite' => 1, + 'testSuffix' => 'limit', + 'otherRequestParameters' => array( + 'filter_limit' => '2', + 'filter_offset' => '1', + ) + ) + ); + $apiToTest[] = array( + $api, + array( + 'date' => "$startDate,$endDate", + 'periods' => array('range'), + 'idSite' => 1, + 'testSuffix' => 'limit', + 'otherRequestParameters' => array( + 'filter_limit' => '2', + 'filter_offset' => '1', + ) + ) + ); + + $apiToTest[] = array( + $api, + array( + 'date' => $startDate, + 'periods' => array('day'), + 'idSite' => 1, + 'testSuffix' => 'ascSortOrder', + 'otherRequestParameters' => array( + 'filter_sort_order' => 'asc', + ) + ) + ); + $apiToTest[] = array( + $api, + array( + 'date' => "$startDate,$endDate", + 'periods' => array('range'), + 'idSite' => 1, + 'testSuffix' => 'ascSortOrder', + 'otherRequestParameters' => array( + 'filter_sort_order' => 'asc', + ) + ) + ); + + $apiToTest[] = array( + $api, + array( + 'date' => $startDate, + 'periods' => array('day'), + 'idSite' => 1, + 'testSuffix' => 'searchByUserId', + 'otherRequestParameters' => array( + 'filter_pattern' => 'user2' + ) + ) + ); + $apiToTest[] = array( + $api, + array( + 'date' => "$startDate,$endDate", + 'periods' => array('range'), + 'idSite' => 1, + 'testSuffix' => 'searchByUserId', + 'otherRequestParameters' => array( + 'filter_pattern' => 'user2' + ) + ) + ); + + return $apiToTest; + } + + public static function getOutputPrefix() + { + return ''; + } + + public static function getPathToTestDirectory() + { + return dirname(__FILE__); + } + +} + +ApiTest::$fixture = new TrackFewVisitsAndCreateUsers(); \ No newline at end of file diff --git a/plugins/UserId/tests/System/expected/test___UserId.getUsers_day.xml b/plugins/UserId/tests/System/expected/test___UserId.getUsers_day.xml new file mode 100644 index 0000000000..19e7d78d39 --- /dev/null +++ b/plugins/UserId/tests/System/expected/test___UserId.getUsers_day.xml @@ -0,0 +1,39 @@ + + + + + 1 + 2 + 3 + 1 + 2 + 1 + 1 + 0 + b3daa77b4c04a955 + + + + 1 + 1 + 1 + 1 + 1 + 0 + 1 + 0 + a1881c06eec96db9 + + + + 1 + 1 + 1 + 1 + 1 + 0 + 1 + 0 + 0b7f849446d33835 + + \ No newline at end of file diff --git a/plugins/UserId/tests/System/expected/test___UserId.getUsers_range.xml b/plugins/UserId/tests/System/expected/test___UserId.getUsers_range.xml new file mode 100644 index 0000000000..184a718ff4 --- /dev/null +++ b/plugins/UserId/tests/System/expected/test___UserId.getUsers_range.xml @@ -0,0 +1,39 @@ + + + + + 31 + 44 + 3 + 12 + 19 + 0 + 30 + 30 + 0b7f849446d33835 + + + + 21 + 29 + 2 + 8 + 13 + 0 + 20 + 20 + a1881c06eec96db9 + + + + 11 + 15 + 2 + 4 + 7 + 0 + 10 + 10 + b3daa77b4c04a955 + + \ No newline at end of file diff --git a/plugins/UserId/tests/System/expected/test_ascSortOrder__UserId.getUsers_day.xml b/plugins/UserId/tests/System/expected/test_ascSortOrder__UserId.getUsers_day.xml new file mode 100644 index 0000000000..9cf17c8b03 --- /dev/null +++ b/plugins/UserId/tests/System/expected/test_ascSortOrder__UserId.getUsers_day.xml @@ -0,0 +1,39 @@ + + + + + 1 + 1 + 1 + 1 + 1 + 0 + 1 + 0 + 0b7f849446d33835 + + + + 1 + 1 + 1 + 1 + 1 + 0 + 1 + 0 + a1881c06eec96db9 + + + + 1 + 2 + 3 + 1 + 2 + 1 + 1 + 0 + b3daa77b4c04a955 + + \ No newline at end of file diff --git a/plugins/UserId/tests/System/expected/test_ascSortOrder__UserId.getUsers_range.xml b/plugins/UserId/tests/System/expected/test_ascSortOrder__UserId.getUsers_range.xml new file mode 100644 index 0000000000..99aa12454a --- /dev/null +++ b/plugins/UserId/tests/System/expected/test_ascSortOrder__UserId.getUsers_range.xml @@ -0,0 +1,39 @@ + + + + + 11 + 15 + 2 + 4 + 7 + 0 + 10 + 10 + b3daa77b4c04a955 + + + + 21 + 29 + 2 + 8 + 13 + 0 + 20 + 20 + a1881c06eec96db9 + + + + 31 + 44 + 3 + 12 + 19 + 0 + 30 + 30 + 0b7f849446d33835 + + \ No newline at end of file diff --git a/plugins/UserId/tests/System/expected/test_limit__UserId.getUsers_day.xml b/plugins/UserId/tests/System/expected/test_limit__UserId.getUsers_day.xml new file mode 100644 index 0000000000..52fcd98131 --- /dev/null +++ b/plugins/UserId/tests/System/expected/test_limit__UserId.getUsers_day.xml @@ -0,0 +1,27 @@ + + + + + 1 + 1 + 1 + 1 + 1 + 0 + 1 + 0 + a1881c06eec96db9 + + + + 1 + 1 + 1 + 1 + 1 + 0 + 1 + 0 + 0b7f849446d33835 + + \ No newline at end of file diff --git a/plugins/UserId/tests/System/expected/test_limit__UserId.getUsers_range.xml b/plugins/UserId/tests/System/expected/test_limit__UserId.getUsers_range.xml new file mode 100644 index 0000000000..9545aadbb7 --- /dev/null +++ b/plugins/UserId/tests/System/expected/test_limit__UserId.getUsers_range.xml @@ -0,0 +1,27 @@ + + + + + 21 + 29 + 2 + 8 + 13 + 0 + 20 + 20 + a1881c06eec96db9 + + + + 11 + 15 + 2 + 4 + 7 + 0 + 10 + 10 + b3daa77b4c04a955 + + \ No newline at end of file diff --git a/plugins/UserId/tests/System/expected/test_searchByUserId__UserId.getUsers_day.xml b/plugins/UserId/tests/System/expected/test_searchByUserId__UserId.getUsers_day.xml new file mode 100644 index 0000000000..ac2a60152a --- /dev/null +++ b/plugins/UserId/tests/System/expected/test_searchByUserId__UserId.getUsers_day.xml @@ -0,0 +1,15 @@ + + + + + 1 + 1 + 1 + 1 + 1 + 0 + 1 + 0 + a1881c06eec96db9 + + \ No newline at end of file diff --git a/plugins/UserId/tests/System/expected/test_searchByUserId__UserId.getUsers_range.xml b/plugins/UserId/tests/System/expected/test_searchByUserId__UserId.getUsers_range.xml new file mode 100644 index 0000000000..ec3854dfb3 --- /dev/null +++ b/plugins/UserId/tests/System/expected/test_searchByUserId__UserId.getUsers_range.xml @@ -0,0 +1,15 @@ + + + + + 21 + 29 + 2 + 8 + 13 + 0 + 20 + 20 + a1881c06eec96db9 + + \ No newline at end of file -- cgit v1.2.3