diff options
author | diosmosis <benaka@piwik.pro> | 2015-05-08 02:58:56 +0300 |
---|---|---|
committer | diosmosis <benaka@piwik.pro> | 2015-06-11 09:42:04 +0300 |
commit | 30ac243f3644bb32eda74b1c38c586ce6c81a7d8 (patch) | |
tree | 33bd0e53649f4bb4c9a05ad19ed0fbdc09d39342 /plugins/PrivacyManager | |
parent | b48e09afd39bed3a598287d4aa8ac2a39c2c4082 (diff) |
Move deleteUnusedLogAction logic to RawLogDao class.
Diffstat (limited to 'plugins/PrivacyManager')
-rw-r--r-- | plugins/PrivacyManager/DimensionMetadataProvider.php | 109 | ||||
-rwxr-xr-x | plugins/PrivacyManager/LogDataPurger.php | 130 | ||||
-rw-r--r-- | plugins/PrivacyManager/tests/Unit/DimensionMetadataProviderTest.php | 124 |
3 files changed, 2 insertions, 361 deletions
diff --git a/plugins/PrivacyManager/DimensionMetadataProvider.php b/plugins/PrivacyManager/DimensionMetadataProvider.php deleted file mode 100644 index 6f2b4e9afd..0000000000 --- a/plugins/PrivacyManager/DimensionMetadataProvider.php +++ /dev/null @@ -1,109 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - */ - -namespace Piwik\Plugins\PrivacyManager; - -use Piwik\Plugin\Dimension\ActionDimension; - -/** - * Provides metadata about dimensions for the LogDataPurger class. - */ -class DimensionMetadataProvider -{ - /** - * Overrids for the result of the getActionReferenceColumnsByTable() method. Exists so Piwik - * instances can be monkey patched, in case there are idaction columns that this class does not - * naturally discover. - * - * @var array - */ - private $actionReferenceColumnsOverride; - - public function __construct(array $actionReferenceColumnsOverride = array()) - { - $this->actionReferenceColumnsOverride = $actionReferenceColumnsOverride; - } - - /** - * Returns a list of idaction column names organized by table name. Uses dimension metadata - * to find idaction columns dynamically. - * - * Note: It is not currently possible to use the Piwik platform to add idaction columns to tables - * other than log_link_visit_action (w/o doing something unsupported), so idaction columns in - * other tables are hard coded. - * - * @return array[] - */ - public function getActionReferenceColumnsByTable() - { - $result = array( - 'log_link_visit_action' => array('idaction_url', - 'idaction_url_ref', - 'idaction_name_ref' - ), - - 'log_conversion' => array('idaction_url'), - - 'log_visit' => array('visit_exit_idaction_url', - 'visit_exit_idaction_name', - 'visit_entry_idaction_url', - 'visit_entry_idaction_name'), - - 'log_conversion_item' => array('idaction_sku', - 'idaction_name', - 'idaction_category', - 'idaction_category2', - 'idaction_category3', - 'idaction_category4', - 'idaction_category5') - ); - - $dimensionIdActionColumns = $this->getVisitActionTableActionReferences(); - $result['log_link_visit_action'] = array_unique( - array_merge($result['log_link_visit_action'], $dimensionIdActionColumns)); - - foreach ($this->actionReferenceColumnsOverride as $table => $columns) { - if (empty($result[$table])) { - $result[$table] = $columns; - } else { - $result[$table] = array_unique(array_merge($result[$table], $columns)); - } - } - - return $result; - } - - private function getVisitActionTableActionReferences() - { - $idactionColumns = array(); - foreach (ActionDimension::getAllDimensions() as $actionDimension) { - if ($this->isActionReference($actionDimension)) { - $idactionColumns[] = $actionDimension->getColumnName(); - } - } - return $idactionColumns; - } - - - /** - * Returns `true` if the column for this dimension is a reference to the `log_action` table (ie, an "idaction column"), - * `false` if otherwise. - * - * @return bool - */ - private function isActionReference(ActionDimension $dimension) - { - try { - $dimension->getActionId(); - - return true; - } catch (\Exception $ex) { - return false; - } - } -} diff --git a/plugins/PrivacyManager/LogDataPurger.php b/plugins/PrivacyManager/LogDataPurger.php index 81b4aaebaa..4705a9091d 100755 --- a/plugins/PrivacyManager/LogDataPurger.php +++ b/plugins/PrivacyManager/LogDataPurger.php @@ -22,19 +22,12 @@ use Piwik\Piwik; */ class LogDataPurger { - const TEMP_TABLE_NAME = 'tmp_log_actions_to_keep'; - /** * The max set of rows each table scan select should query at one time. */ public static $selectSegmentSize = 100000; /** - * @param DimensionMetadataProvider - */ - private $dimensionMetadataProvider; - - /** * TODO * * @var LogPurger @@ -58,9 +51,8 @@ class LogDataPurger /** * Constructor. */ - public function __construct(DimensionMetadataProvider $dimensionMetadataProvider, LogPurger $logPurger, RawLogDao $rawLogDao) + public function __construct(LogPurger $logPurger, RawLogDao $rawLogDao) { - $this->dimensionMetadataProvider = $dimensionMetadataProvider; $this->logPurger = $logPurger; $this->rawLogDao = $rawLogDao; } @@ -96,7 +88,7 @@ class LogDataPurger // delete unused actions from the log_action table (but only if we can lock tables) if (Db::isLockPrivilegeGranted()) { - $this->purgeUnusedLogActions(); // TODO: move actual code to DAO/service in core + $this->rawLogDao->deleteUnusedLogActions(); } else { $logMessage = get_class($this) . ": LOCK TABLES privilege not granted; skipping unused actions purge"; Log::warning($logMessage); @@ -143,29 +135,6 @@ class LogDataPurger } /** - * Safely delete all unused log_action rows. - */ - private function purgeUnusedLogActions() - { - $this->createTempTable(); - - // get current max ID in log tables w/ idaction references. - $maxIds = $this->getMaxIdsInLogTables(); - - // do large insert (inserting everything before maxIds) w/o locking tables... - $this->insertActionsToKeep($maxIds, $deleteOlderThanMax = true); - - // ... then do small insert w/ locked tables to minimize the amount of time tables are locked. - $this->lockLogTables(); - $this->insertActionsToKeep($maxIds, $deleteOlderThanMax = false); - - // delete before unlocking tables so there's no chance a new log row that references an - // unused action will be inserted. - $this->deleteUnusedActions(); - Db::unlockAllTables(); - } - - /** * get highest idVisit to delete rows from * @return string */ @@ -198,101 +167,6 @@ class LogDataPurger return (int) Db::fetchOne($sql, array($maxIdVisit)); } - private function createTempTable() - { - $sql = "CREATE TEMPORARY TABLE " . Common::prefixTable(self::TEMP_TABLE_NAME) . " ( - idaction INT(11), - PRIMARY KEY (idaction) - )"; - Db::query($sql); - } - - private function getMaxIdsInLogTables() - { - $tables = array('log_conversion', 'log_link_visit_action', 'log_visit', 'log_conversion_item'); - $idColumns = $this->getTableIdColumns(); - - $result = array(); - foreach ($tables as $table) { - $idCol = $idColumns[$table]; - $result[$table] = Db::fetchOne("SELECT MAX($idCol) FROM " . Common::prefixTable($table)); - } - - return $result; - } - - private function insertActionsToKeep($maxIds, $olderThan = true) - { - $tempTableName = Common::prefixTable(self::TEMP_TABLE_NAME); - - $idColumns = $this->getTableIdColumns(); - $idActionColumnsByTable = $this->dimensionMetadataProvider->getActionReferenceColumnsByTable(); - foreach ($idActionColumnsByTable as $table => $columns) { - $idCol = $idColumns[$table]; - - foreach ($columns as $col) { - $select = "SELECT $col FROM " . Common::prefixTable($table) . " WHERE $idCol >= ? AND $idCol < ?"; - $sql = "INSERT IGNORE INTO $tempTableName $select"; - - if ($olderThan) { - $start = 0; - $finish = $maxIds[$table]; - } else { - $start = $maxIds[$table]; - $finish = Db::fetchOne("SELECT MAX($idCol) FROM " . Common::prefixTable($table)); - } - - Db::segmentedQuery($sql, $start, $finish, self::$selectSegmentSize); - } - } - - // allow code to be executed after data is inserted. for concurrency testing purposes. - if ($olderThan) { - /** - * @ignore - */ - Piwik::postEvent("LogDataPurger.ActionsToKeepInserted.olderThan"); - } else { - /** - * @ignore - */ - Piwik::postEvent("LogDataPurger.ActionsToKeepInserted.newerThan"); - } - } - - private function lockLogTables() - { - Db::lockTables( - $readLocks = Common::prefixTables('log_conversion', - 'log_link_visit_action', - 'log_visit', - 'log_conversion_item'), - $writeLocks = Common::prefixTables('log_action') - ); - } - - private function deleteUnusedActions() - { - list($logActionTable, $tempTableName) = Common::prefixTables("log_action", self::TEMP_TABLE_NAME); - - $deleteSql = "DELETE LOW_PRIORITY QUICK IGNORE $logActionTable - FROM $logActionTable - LEFT JOIN $tempTableName tmp ON tmp.idaction = $logActionTable.idaction - WHERE tmp.idaction IS NULL"; - - Db::query($deleteSql); - } - - private function getTableIdColumns() - { - return array( - 'log_link_visit_action' => 'idlink_va', - 'log_conversion' => 'idvisit', - 'log_visit' => 'idvisit', - 'log_conversion_item' => 'idvisit' - ); - } - // let's hardcode, since these are not dynamically created tables public static function getDeleteTableLogTables() { diff --git a/plugins/PrivacyManager/tests/Unit/DimensionMetadataProviderTest.php b/plugins/PrivacyManager/tests/Unit/DimensionMetadataProviderTest.php deleted file mode 100644 index 7c015a9206..0000000000 --- a/plugins/PrivacyManager/tests/Unit/DimensionMetadataProviderTest.php +++ /dev/null @@ -1,124 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - */ - -namespace Piwik\Plugins\PrivacyManager\tests\Unit; - -use Piwik\Plugins\PrivacyManager\DimensionMetadataProvider; -use Piwik\Tests\Framework\TestCase\UnitTestCase; -use Piwik\Plugin\Manager as PluginManager; - -class DimensionMetadataProviderTest extends UnitTestCase -{ - public function setUp() - { - parent::setUp(); - - /** @var PluginManager $manager */ - $manager = $this->environment->getContainer()->get('Piwik\Plugin\Manager'); - $manager->loadPlugins(array('Events', 'Contents')); - } - - public function test_getActionReferenceColumnsByTable_DetectsActionReferenceDimensions_AndIncludesHardocdedColumns() - { - $dimensionMetadataProvider = new DimensionMetadataProvider(); - - $actualColumns = $dimensionMetadataProvider->getActionReferenceColumnsByTable(); - - $expectedColumns = array( - 'log_link_visit_action' => array( - 'idaction_url', - 'idaction_url_ref', - 'idaction_name_ref', - 'idaction_event_action', - 'idaction_event_category', - 'idaction_name', - 'idaction_content_interaction', - 'idaction_content_name', - 'idaction_content_piece', - 'idaction_content_target' - ), - 'log_conversion' => array( - 'idaction_url', - ), - 'log_visit' => array( - 'visit_exit_idaction_url', - 'visit_exit_idaction_name', - 'visit_entry_idaction_url', - 'visit_entry_idaction_name', - ), - 'log_conversion_item' => array( - 'idaction_sku', - 'idaction_name', - 'idaction_category', - 'idaction_category2', - 'idaction_category3', - 'idaction_category4', - 'idaction_category5', - ), - ); - - $this->assertEquals($expectedColumns, $actualColumns); - } - - public function test_getActionReferenceColumnsByTable_AppliesOverrideColumnsCorrectly_WithoutAllowingDuplicates() - { - $dimensionMetadataProvider = new DimensionMetadataProvider(array( - 'log_link_visit_action' => array('idaction_url', - 'idaction_event_category' - ), - - 'log_conversion' => array(), - - 'log_conversion_item' => array('some_unknown_idaction_column'), - - 'log_custom_table' => array('some_column1', 'some_column2') - )); - - $actualColumns = $dimensionMetadataProvider->getActionReferenceColumnsByTable(); - - $expectedColumns = array( - 'log_link_visit_action' => array( - 'idaction_url', - 'idaction_url_ref', - 'idaction_name_ref', - 'idaction_event_action', - 'idaction_event_category', - 'idaction_name', - 'idaction_content_interaction', - 'idaction_content_name', - 'idaction_content_piece', - 'idaction_content_target' - ), - 'log_conversion' => array( - 'idaction_url', - ), - 'log_visit' => array( - 'visit_exit_idaction_url', - 'visit_exit_idaction_name', - 'visit_entry_idaction_url', - 'visit_entry_idaction_name', - ), - 'log_conversion_item' => array( - 'idaction_sku', - 'idaction_name', - 'idaction_category', - 'idaction_category2', - 'idaction_category3', - 'idaction_category4', - 'idaction_category5', - 'some_unknown_idaction_column' - ), - 'log_custom_table' => array( - 'some_column1', - 'some_column2' - ) - ); - - $this->assertEquals($expectedColumns, $actualColumns); - } -}
\ No newline at end of file |