diff options
author | Thomas Steur <thomas.steur@gmail.com> | 2015-03-31 00:40:53 +0300 |
---|---|---|
committer | Thomas Steur <thomas.steur@gmail.com> | 2015-03-31 05:27:06 +0300 |
commit | 12d9b392a9f2a8a867d04b7c3889cfc261b783b1 (patch) | |
tree | 2146491c239b3e63a3cb5ad11278f0c0af4a16da | |
parent | e4d1c8bdbe2f6adbe6c70f3a1289af4d66818348 (diff) |
A chunk implementation that is much simpler and makes more sense. Everything is now in the ArchiveWriter + Selector
-rw-r--r-- | .travis.yml | 4 | ||||
-rw-r--r-- | core/Archive.php | 77 | ||||
-rw-r--r-- | core/Archive/Chunk.php | 61 | ||||
-rw-r--r-- | core/ArchiveProcessor.php | 33 | ||||
-rw-r--r-- | core/DataAccess/ArchiveSelector.php | 83 | ||||
-rw-r--r-- | core/DataAccess/ArchiveWriter.php | 32 | ||||
-rw-r--r-- | tests/PHPUnit/Integration/ArchiveTest.php | 76 | ||||
-rwxr-xr-x | tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTestsTest.php | 2 | ||||
-rwxr-xr-x | tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php | 2 | ||||
-rw-r--r-- | tests/PHPUnit/Unit/Archive/ChunkTest.php | 45 | ||||
-rw-r--r-- | tests/PHPUnit/Unit/ArchiveProcessorTest.php | 129 | ||||
-rw-r--r-- | tests/PHPUnit/Unit/DataAccess/ArchiveWriterTest.php | 65 |
12 files changed, 233 insertions, 376 deletions
diff --git a/.travis.yml b/.travis.yml index 710e63fa52..78def33e1c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -48,9 +48,9 @@ matrix: - php: 5.3.3 env: TEST_SUITE=IntegrationTests MYSQL_ADAPTER=PDO_MYSQL - php: 5.3.3 - env: TEST_SUITE=UnitTests MYSQL_ADAPTER=PDO_MYSQL - - php: 5.3.3 env: TEST_SUITE=AllTests MYSQL_ADAPTER=PDO_MYSQL + - php: 5.3.3 + env: TEST_SUITE=UnitTests MYSQL_ADAPTER=PDO_MYSQL - php: hhvm env: TEST_SUITE=SystemTests MYSQL_ADAPTER=PDO_MYSQL - php: hhvm diff --git a/core/Archive.php b/core/Archive.php index c2316f21f0..e36619d119 100644 --- a/core/Archive.php +++ b/core/Archive.php @@ -595,24 +595,23 @@ class Archive $archiveNames = array($archiveNames); } - $chunk = new Chunk(); - - $chunks = array(); - // apply idSubtable if ($idSubtable !== null && $idSubtable != self::ID_SUBTABLE_LOAD_ALL_SUBTABLES ) { - foreach ($archiveNames as &$name) { - // to be backwards compatibe we need to look for the exact idSubtable blob and for the chunk - // that stores the subtables (a chunk stores many blobs in one blob) - $chunks[] = $this->appendIdSubtable($name, $chunk->getBlobIdForTable($idSubtable)); - $name = $this->appendIdsubtable($name, $idSubtable); + // this is also done in ArchiveSelector. It should be actually only done in ArchiveSelector but DataCollection + // does require to have the subtableId appended. Needs to be changed in refactoring to have it only in one + // place. + $dataNames = array(); + foreach ($archiveNames as $name) { + $dataNames[] = $this->appendIdsubtable($name, $idSubtable); } + } else { + $dataNames = $archiveNames; } $result = new Archive\DataCollection( - $archiveNames, $archiveDataType, $this->params->getIdSites(), $this->params->getPeriods(), $defaultRow = null); + $dataNames, $archiveDataType, $this->params->getIdSites(), $this->params->getPeriods(), $defaultRow = null); $archiveIds = $this->getArchiveIds($archiveNames); @@ -620,14 +619,7 @@ class Archive return $result; } - if (!empty($chunks)) { - // we cannot merge them to $archiveNames further up when generating the chunk names as the DataCollection - // would think that eg "chunk_0" is a subtable. - $archiveNames = array_merge($archiveNames, $chunks); - } - - $loadAllSubtables = $idSubtable == self::ID_SUBTABLE_LOAD_ALL_SUBTABLES; - $archiveData = ArchiveSelector::getArchiveData($archiveIds, $archiveNames, $archiveDataType, $loadAllSubtables); + $archiveData = ArchiveSelector::getArchiveData($archiveIds, $archiveNames, $archiveDataType, $idSubtable); foreach ($archiveData as $row) { // values are grouped by idsite (site ID), date1-date2 (date range), then name (field name) @@ -635,58 +627,20 @@ class Archive $periodStr = $row['date1'] . "," . $row['date2']; if ($archiveDataType == 'numeric') { - $value = $this->formatNumericValue($row['value']); + $row['value'] = $this->formatNumericValue($row['value']); } else { - $value = $this->uncompress($row['value']); $result->addMetadata($idSite, $periodStr, 'ts_archived', $row['ts_archived']); } $resultRow = & $result->get($idSite, $periodStr); - if ($chunk->isRecordNameAChunk($row['name'])) { - // one combined blob for all subtables - $value = unserialize($value); - - if (is_array($value)) { - $rawName = $chunk->getRecordNameWithoutChunkAppendix($row['name']); // eg PluginName_ArchiveName - - if ($loadAllSubtables) { - $this->moveAllBlobsWithinChunkToResultRow($resultRow, $rawName, $value); - } else { - $this->moveNeededBlobWithinChunkToResultRow($resultRow, $rawName, $idSubtable, $value); - } - } - - } else { - // one blob per datatable or subtable - $resultRow[$row['name']] = $value; - } - + // one blob per datatable or subtable + $resultRow[$row['name']] = $row['value']; } return $result; } - private function moveNeededBlobWithinChunkToResultRow(&$resultRow, $recordName, $idSubtable, $chunk) - { - $subtableRecordName = $this->appendIdSubtable($recordName, $idSubtable); - - if (array_key_exists($idSubtable, $chunk)) { - $resultRow[$subtableRecordName] = $chunk[$idSubtable]; - } else { - $resultRow[$subtableRecordName] = 0; - unset($resultRow['_metadata']); - } - } - - private function moveAllBlobsWithinChunkToResultRow(&$resultRow, $recordName, $chunk) - { - foreach ($chunk as $subtableId => $val) { - $subtableRecordName = $this->appendIdSubtable($recordName, $subtableId); - $resultRow[$subtableRecordName] = $val; - } - } - /** * Returns archive IDs for the sites, periods and archive names that are being * queried. This function will use the idarchive cache if it has the right data, @@ -865,11 +819,6 @@ class Archive return round((float)$value, 2); } - private function uncompress($data) - { - return @gzuncompress($data); - } - /** * Initializes the archive ID cache ($this->idarchives) for a particular 'done' flag. * diff --git a/core/Archive/Chunk.php b/core/Archive/Chunk.php index 5b0c04f423..70afec446f 100644 --- a/core/Archive/Chunk.php +++ b/core/Archive/Chunk.php @@ -12,11 +12,11 @@ namespace Piwik\Archive; use Piwik\DataTable; /** - * This class is used to hold and transform archive data for the Archive class. + * This class is used to split blobs of DataTables into chunks. Each blob used to be stored under one blob in the + * archive table. For better efficiency we do now combine multiple DataTable into one blob entry. * - * Archive data is loaded into an instance of this type, can be indexed by archive - * metadata (such as the site ID, period string, etc.), and can be transformed into - * DataTable and Map instances. + * Chunks are identified by having the recordName $recordName_chunk_0_99, $recordName_chunk_100_199 (this chunk stores + * the subtable 100-199). */ class Chunk { @@ -24,58 +24,52 @@ class Chunk const NUM_TABLES_IN_CHUNK = 100; /** - * Get's the BlobId to use for a given tableId/subtableId. + * Get's the record name to use for a given tableId/subtableId. * - * @param int $tableId eg '5' for tableId '5' - * @return string eg 'chunk_0' as the table should be within this chunk. + * @param string $recordName eg 'Actions_ActionsUrl' + * @param int $tableId eg '5' for tableId '5' + * @return string eg 'Actions_ActionsUrl_chunk_0_99' as the table should be stored under this blob id. */ - public function getBlobIdForTable($tableId) + public function getRecordNameForTableId($recordName, $tableId) { $chunk = (floor($tableId / self::NUM_TABLES_IN_CHUNK)); $start = $chunk * self::NUM_TABLES_IN_CHUNK; $end = $start + self::NUM_TABLES_IN_CHUNK - 1; - return self::ARCHIVE_APPENDIX_SUBTABLES . '_' . $start . '_' . $end; + return $recordName . $this->getAppendix() . $start . '_' . $end; } /** - * Checks whether a BlobId belongs to a chunk or not. - * @param string|int $blobId eg "1" (for subtableId "1", not a chunk) or "chunk_4" which is a blob id for a chunk - * @return bool true of it starts with "chunk_" - */ - public function isBlobIdAChunk($blobId) - { - return strpos($blobId, self::ARCHIVE_APPENDIX_SUBTABLES . '_') === 0; - } - - /** - * Moves the given blobs into chunks and assigns a proper blobId. + * Moves the given blobs into chunks and assigns a proper record name containing the chunk number. * - * @param array $blobs An array containg a mapping of tableIds to blobs. Eg array(0 => 'blob', 1 => 'subtableBlob', ...) - * @return array An array where each blob is moved into a chunk, indexed by BlobId. - * eg array('chunk_0' => array(0 => 'blob', 1 => 'subtableBlob', ...), 'chunk_1' => array(...)) + * @param string $recordName The original archive record name, eg 'Actions_ActionsUrl' + * @param array $blobs An array containg a mapping of tableIds to blobs. Eg array(0 => 'blob', 1 => 'subtableBlob', ...) + * @return array An array where each blob is moved into a chunk, indexed by recordNames. + * eg array('Actions_ActionsUrl_chunk_0_99' => array(0 => 'blob', 1 => 'subtableBlob', ...), + * 'Actions_ActionsUrl_chunk_100_199' => array(...)) */ - public function moveArchiveBlobsIntoChunks($blobs) + public function moveArchiveBlobsIntoChunks($recordName, $blobs) { $chunks = array(); foreach ($blobs as $tableId => $blob) { - $blobId = $this->getBlobIdForTable($tableId); + $name = $this->getRecordNameForTableId($recordName, $tableId); - if (!array_key_exists($blobId, $chunks)) { - $chunks[$blobId] = array(); + if (!array_key_exists($name, $chunks)) { + $chunks[$name] = array(); } - $chunks[$blobId][$tableId] = $blob; + $chunks[$name][$tableId] = $blob; } return $chunks; } /** - * Detects whether a recordName like 'Actions_ActionUrls_chunk_5' or 'Actions_ActionUrls' belongs to a chunk or not. + * Detects whether a recordName like 'Actions_ActionUrls_chunk_0_99' or 'Actions_ActionUrls' belongs to a + * chunk or not. * - * To be a valid recordName that belongs to a chunk it must end with '_chunk_NUMERIC'. + * To be a valid recordName that belongs to a chunk it must end with '_chunk_NUMERIC_NUMERIC'. * * @param string $recordName * @return bool @@ -102,7 +96,7 @@ class Chunk } /** - * When having a record like 'Actions_ActionUrls_chunk_5" it will return the raw recordName 'Actions_ActionUrls'. + * When having a record like 'Actions_ActionUrls_chunk_0_99" it will return the raw recordName 'Actions_ActionUrls'. * * @param string $recordName * @return string @@ -122,6 +116,11 @@ class Chunk return substr($recordName, 0, $posAppendix); } + /** + * Returns the string that is appended to the original record name. This appendix identifes a record name is a + * chunk. + * @return string + */ public function getAppendix() { return '_' . self::ARCHIVE_APPENDIX_SUBTABLES . '_'; diff --git a/core/ArchiveProcessor.php b/core/ArchiveProcessor.php index d3f8b29837..ccfa939d94 100644 --- a/core/ArchiveProcessor.php +++ b/core/ArchiveProcessor.php @@ -9,7 +9,6 @@ namespace Piwik; use Exception; -use Piwik\Archive\Chunk; use Piwik\Archive\DataTableFactory; use Piwik\ArchiveProcessor\Parameters; use Piwik\ArchiveProcessor\Rules; @@ -327,36 +326,14 @@ class ArchiveProcessor * @param string $name The name of the record, eg, 'Referrers_type'. * @param string|array $values A blob string or an array of blob strings. If an array * is used, the first element in the array will be inserted - * with the `$name` name. The others will be splitted into chunks of 100 and inserted - * with `$name . 'chunk_' . $index` as the record name (where $index is - * the index of the chunk). All subtables within one chunk will be serialized as an - * array where the index is the subtableId. + * with the `$name` name. The others will be inserted with + * `$name . '_' . $index` as the record name (where $index is + * the index of the blob record in `$values`). * @api */ public function insertBlobRecord($name, $values) { - $newInsert = array(); - - if (is_array($values)) { - if (isset($values[0])) { - // we always store the root table in a single blob for fast access - $newInsert[0] = $values[0]; - unset($values[0]); - } - - if (!empty($values)) { - // we move all subtables into chunks - $chunk = new Chunk(); - $chunks = $chunk->moveArchiveBlobsIntoChunks($values); - foreach ($chunks as $index => $subtables) { - $newInsert[$index] = serialize($subtables); - } - } - } else { - $newInsert = $values; - } - - $this->archiveWriter->insertBlobRecord($name, $newInsert); + $this->archiveWriter->insertBlobRecord($name, $values); } /** @@ -596,4 +573,4 @@ class ArchiveProcessor return $metrics; } -} +}
\ No newline at end of file diff --git a/core/DataAccess/ArchiveSelector.php b/core/DataAccess/ArchiveSelector.php index aaca5c30ad..2d38ab9146 100644 --- a/core/DataAccess/ArchiveSelector.php +++ b/core/DataAccess/ArchiveSelector.php @@ -221,21 +221,23 @@ class ArchiveSelector * @param array $recordNames The names of the data to retrieve (ie, nb_visits, nb_actions, etc.). * Note: You CANNOT pass multiple recordnames if $loadAllSubtables=true. * @param string $archiveDataType The archive data type (either, 'blob' or 'numeric'). - * @param bool $loadAllSubtables Whether to pre-load all subtables + * @param int $idSubtable * @throws Exception * @return array */ - public static function getArchiveData($archiveIds, $recordNames, $archiveDataType, $loadAllSubtables) + public static function getArchiveData($archiveIds, $recordNames, $archiveDataType, $idSubtable) { + $chunk = new Archive\Chunk(); + // create the SQL to select archive data - $inNames = Common::getSqlStringFieldsArray($recordNames); + $loadAllSubtables = $idSubtable == Archive::ID_SUBTABLE_LOAD_ALL_SUBTABLES; if ($loadAllSubtables) { + $name = reset($recordNames); // select blobs w/ name like "$name_[0-9]+" w/o using RLIKE $nameEnd = strlen($name) + 1; $nameEndAppendix = $nameEnd + 1; - $chunk = new Archive\Chunk(); $appendix = $chunk->getAppendix(); $lenAppendix = strlen($appendix); @@ -246,8 +248,23 @@ class ArchiveSelector $whereNameIs = "(name = ? OR (name LIKE ? AND ( $checkForChunkBlob OR $checkForSubtableId ) ))"; $bind = array($name, $name . '%'); } else { + + if ($idSubtable === null) { + // select root table or specific record names + $bind = array_values($recordNames); + } else { + // select a subtable id + $bind = array(); + foreach ($recordNames as $recordName) { + // to be backwards compatibe we need to look for the exact idSubtable blob and for the chunk + // that stores the subtables (a chunk stores many blobs in one blob) + $bind[] = $chunk->getRecordNameForTableId($recordName, $idSubtable); + $bind[] = self::appendIdSubtable($recordName, $idSubtable); + } + } + + $inNames = Common::getSqlStringFieldsArray($bind); $whereNameIs = "name IN ($inNames)"; - $bind = array_values($recordNames); } $getValuesSql = "SELECT value, name, idsite, date1, date2, ts_archived @@ -266,7 +283,8 @@ class ArchiveSelector // $period = "2009-01-04,2009-01-04", $date = Date::factory(substr($period, 0, 10)); - if ($archiveDataType == 'numeric') { + $isNumeric = $archiveDataType == 'numeric'; + if ($isNumeric) { $table = ArchiveTableCreator::getNumericTable($date); } else { $table = ArchiveTableCreator::getBlobTable($date); @@ -276,13 +294,64 @@ class ArchiveSelector $dataRows = Db::fetchAll($sql, $bind); foreach ($dataRows as $row) { - $rows[] = $row; + if ($isNumeric) { + $rows[] = $row; + } else { + + $row['value'] = self::uncompress($row['value']); + + if ($chunk->isRecordNameAChunk($row['name'])) { + $blobs = unserialize($row['value']); + + if (!is_array($blobs)) { + continue; + } + + // $rawName = eg 'PluginName_ArchiveName' + $rawName = $chunk->getRecordNameWithoutChunkAppendix($row['name']); + + if ($loadAllSubtables) { + foreach ($blobs as $subtableId => $blob) { + $rows[] = array( + 'name' => self::appendIdSubtable($rawName, $subtableId), + 'value' => $blob, + 'idsite' => $row['idsite'], + 'date1' => $row['date1'], + 'date2' => $row['date2'], + 'ts_archived' => $row['ts_archived'], + ); + } + } elseif (array_key_exists($idSubtable, $blobs)) { + $rows[] = array( + 'name' => self::appendIdSubtable($rawName, $idSubtable), + 'value' => $blobs[$idSubtable], + 'idsite' => $row['idsite'], + 'date1' => $row['date1'], + 'date2' => $row['date2'], + 'ts_archived' => $row['ts_archived'], + ); + } + + } else { + $rows[] = $row; + } + } } } return $rows; } + private static function appendIdSubtable($recordName, $id) + { + return $recordName . "_" . $id; + } + + private static function uncompress($data) + { + return @gzuncompress($data); + } + /** * Returns the SQL condition used to find successfully completed archives that * this instance is querying for. diff --git a/core/DataAccess/ArchiveWriter.php b/core/DataAccess/ArchiveWriter.php index 7ca3b3cfe7..56c0927496 100644 --- a/core/DataAccess/ArchiveWriter.php +++ b/core/DataAccess/ArchiveWriter.php @@ -77,27 +77,33 @@ class ArchiveWriter /** * @param string $name - * @param string[] $values + * @param string|string[] $values A blob string or an array of blob strings. If an array + * is used, the first element in the array will be inserted + * with the `$name` name. The others will be splitted into chunks. All subtables + * within one chunk will be serialized as an array where the index is the + * subtableId. */ public function insertBlobRecord($name, $values) { if (is_array($values)) { - $chunk = new Chunk(); $clean = array(); - foreach ($values as $id => $value) { - // for the parent Table we keep the name - // for example for the Table of searchEngines we keep the name 'referrer_search_engine' - // but for the child table of 'Google' which has the ID = 9 the name would be 'referrer_search_engine_9' - $newName = $name; - if ($id != 0 || $chunk->isBlobIdAChunk($id)) { - //FIXMEA: refactor - $newName = $name . '_' . $id; - } - $value = $this->compress($value); - $clean[] = array($newName, $value); + if (isset($values[0])) { + // we always store the root table in a single blob for fast access + $clean[] = array($name, $this->compress($values[0])); + unset($values[0]); } + + if (!empty($values)) { + // we move all subtables into chunks + $chunk = new Chunk(); + $chunks = $chunk->moveArchiveBlobsIntoChunks($name, $values); + foreach ($chunks as $index => $subtables) { + $clean[] = array($index, $this->compress(serialize($subtables))); + } + } + $this->insertBulkRecords($clean); return; } diff --git a/tests/PHPUnit/Integration/ArchiveTest.php b/tests/PHPUnit/Integration/ArchiveTest.php index f217fdaca7..906cfe40c9 100644 --- a/tests/PHPUnit/Integration/ArchiveTest.php +++ b/tests/PHPUnit/Integration/ArchiveTest.php @@ -196,18 +196,18 @@ class ArchiveTest extends IntegrationTestCase */ public function test_ArchiveBlob_ShouldBeFindBlobs_WithinDifferentChunks($idSubtable, $expectedBlob) { + $recordName = 'Actions_Actions'; + $chunk = new Chunk(); - $chunk5 = $chunk->getBlobIdForTable($subtableId = 5); - $chunk152 = $chunk->getBlobIdForTable($subtableId = 152); - $chunk399 = $chunk->getBlobIdForTable($subtableId = 399); + $chunk5 = $chunk->getRecordNameForTableId($recordName, $subtableId = 5); + $chunk152 = $chunk->getRecordNameForTableId($recordName, $subtableId = 152); + $chunk399 = $chunk->getRecordNameForTableId($recordName, $subtableId = 399); $this->createArchiveBlobEntry('2013-01-02', array( - 'Actions_Actions' => array( - 0 => 'actions_02', - $chunk5 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')), - $chunk152 => serialize(array(151 => 'actionsSubtable151', 152 => 'actionsSubtable152')), - $chunk399 => serialize(array(399 => 'actionsSubtable399')), - ) + $recordName => 'actions_02', + $chunk5 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')), + $chunk152 => serialize(array(151 => 'actionsSubtable151', 152 => 'actionsSubtable152')), + $chunk399 => serialize(array(399 => 'actionsSubtable399')) )); $archive = $this->getArchive('day', '2013-01-02,2013-01-02'); @@ -231,49 +231,39 @@ class ArchiveTest extends IntegrationTestCase private function createManyDifferentArchiveBlobs() { + $recordName1 = 'Actions_Actions'; + $recordName2 = 'Actions_Actionsurl'; + $chunk = new Chunk(); - $chunk0 = $chunk->getBlobIdForTable(0); + $chunk0_1 = $chunk->getRecordNameForTableId($recordName1, 0); + $chunk0_2 = $chunk->getRecordNameForTableId($recordName2, 0); $this->createArchiveBlobEntry('2013-01-01', array( - 'Actions_Actionsurl' => array( - 0 => 'test01' - ) + $recordName2 => 'test01' )); $this->createArchiveBlobEntry('2013-01-02', array( - 'Actions_Actionsurl' => array( - 0 => 'test02', - 1 => 'test1', - 2 => 'test2' - ), - 'Actions_Actions' => array( - 0 => 'actions_02', - $chunk0 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')) - ) + $recordName2 => 'test02', + $recordName2 . '_1' => 'test1', // testing BC where each subtable was stored seperately + $recordName2 . '_2' => 'test2', // testing BC + $recordName1 => 'actions_02', + $chunk0_1 => serialize(array(1 => 'actionsSubtable1', 2 => 'actionsSubtable2', 5 => 'actionsSubtable5')) )); $this->createArchiveBlobEntry('2013-01-03', array( - 'Actions_Actionsurl' => array( - 0 => 'test03', - $chunk0 => serialize(array(1 => 'subtable1', 2 => 'subtable2', 5 => 'subtable5')) - ), - 'Actions_Actions' => array( - 0 => 'actions_03', - 1 => 'actionsTest1', - 2 => 'actionsTest2') - ) - ); + $recordName2 => 'test03', + $chunk0_2 => serialize(array(1 => 'subtable1', 2 => 'subtable2', 5 => 'subtable5')), + $recordName1 => 'actions_03', + $recordName1 . '_1' => 'actionsTest1', + $recordName1 . '_2' => 'actionsTest2' + )); $this->createArchiveBlobEntry('2013-01-04', array( - 'Actions_Actionsurl' => array( - 0 => 'test04', - 5 => 'subtable45', - 6 => 'subtable6') - ) - ); + $recordName2 => 'test04', + $recordName2 . '_5' => 'subtable45', + $recordName2 . '_6' => 'subtable6' + )); $this->createArchiveBlobEntry('2013-01-06', array( - 'Actions_Actionsurl' => array( - 0 => 'test06', - $chunk0 => serialize(array())) - ) - ); + $recordName2 => 'test06', + $chunk0_2 => serialize(array()) + )); } private function assertArchiveBlob(PiwikArchive\DataCollection $dataCollection, $date, $expectedBlob) diff --git a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTestsTest.php b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTestsTest.php index 87e0a8294f..d879c07fb7 100755 --- a/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTestsTest.php +++ b/tests/PHPUnit/System/OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTestsTest.php @@ -183,7 +183,7 @@ class OneVisitorOneWebsiteSeveralDaysDateRangeArchivingTest extends SystemTestCa ); $chunk = new Chunk(); foreach ($tests as $table => $expectedNumSubtables) { - $chunkAppendix = $chunk->getBlobIdForTable(0); + $chunkAppendix = $chunk->getRecordNameForTableId(0); $sql = "SELECT value FROM " . Common::prefixTable($table) . " WHERE period = " . Piwik::$idPeriods['range'] . " and `name` ='Actions_actions_url_$chunkAppendix'"; $blob = Db::get()->fetchOne($sql); $blob = gzuncompress($blob); diff --git a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php index 6c75fb32d2..a2c1d06672 100755 --- a/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php +++ b/tests/PHPUnit/System/TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest.php @@ -117,7 +117,7 @@ class TwoVisitsWithCustomVariablesSegmentMatchVisitorTypeTest extends SystemTest public function test_checkArchiveRecords_shouldMergeSubtablesIntoOneRow() { $chunk = new Chunk(); - $chunkBlobId = $chunk->getBlobIdForTable(0); + $chunkBlobId = $chunk->getRecordNameForTableId(0); $tests = array( 'archive_blob_2010_01' => array( diff --git a/tests/PHPUnit/Unit/Archive/ChunkTest.php b/tests/PHPUnit/Unit/Archive/ChunkTest.php index 29904e7439..dcab94c267 100644 --- a/tests/PHPUnit/Unit/Archive/ChunkTest.php +++ b/tests/PHPUnit/Unit/Archive/ChunkTest.php @@ -24,6 +24,8 @@ class ChunkTest extends UnitTestCase */ private $chunk; + private $recordName = 'Actions_ActionsUrl'; + public function setUp() { parent::setUp(); @@ -31,14 +33,14 @@ class ChunkTest extends UnitTestCase } /** - * @dataProvider getBlobIdForTableDataProvider + * @dataProvider getRecordNameForTableIdDataProvider */ - public function test_getBlobIdForTable_shouldSplitChunksIntoBitsOf100($expectedChunk, $tableId) + public function test_getRecordNameForTableId_shouldSplitChunksIntoBitsOf100($expectedChunk, $tableId) { - $this->assertEquals('chunk_' . $expectedChunk, $this->chunk->getBlobIdForTable($tableId)); + $this->assertEquals($this->recordName . '_chunk_' . $expectedChunk, $this->chunk->getRecordNameForTableId($this->recordName, $tableId)); } - public function getBlobIdForTableDataProvider() + public function getRecordNameForTableId() { return array( array($expectedChunk = '0_99', $tableId = 0), @@ -57,31 +59,6 @@ class ChunkTest extends UnitTestCase } /** - * @dataProvider isBlobIdAChunkDataProvider - */ - public function test_isBlobIdAChunk($isChunk, $blobId) - { - $this->assertSame($isChunk, $this->chunk->isBlobIdAChunk($blobId)); - } - - public function isBlobIdAChunkDataProvider() - { - return array( - array($isChunk = true, $blobId = 'chunk_0_99'), - array(true, 'chunk_100_199'), - // following 2 are not really a chunk but we accept it as such for simpler/faster implementation - array(true, 'chunk_0'), - array(true, 'chunk_999'), - array(false, 'chunk0'), - array(false, 'chunk999'), - array(false, '0'), - array(false, '5'), - array(false, 5), - array(false, '_5'), - ); - } - - /** * @dataProvider isRecordNameAChunkDataProvider */ public function test_isRecordNameAChunk_shouldSplitChunksIntoBitsOf100($isChunk, $recordName) @@ -109,7 +86,7 @@ class ChunkTest extends UnitTestCase public function test_moveArchiveBlobsIntoChunks_NoChunksGiven() { - $this->assertSame(array(), $this->chunk->moveArchiveBlobsIntoChunks(array())); + $this->assertSame(array(), $this->chunk->moveArchiveBlobsIntoChunks($this->recordName, array())); } /** @@ -119,12 +96,12 @@ class ChunkTest extends UnitTestCase { $array = array_fill(0, 245, 'test'); $expected = array( - 'chunk_0_99' => array_fill(0, Chunk::NUM_TABLES_IN_CHUNK, 'test'), - 'chunk_100_199' => array_fill(100, Chunk::NUM_TABLES_IN_CHUNK, 'test'), - 'chunk_200_299' => array_fill(200, 45, 'test'), + $this->recordName . '_chunk_0_99' => array_fill(0, Chunk::NUM_TABLES_IN_CHUNK, 'test'), + $this->recordName . '_chunk_100_199' => array_fill(100, Chunk::NUM_TABLES_IN_CHUNK, 'test'), + $this->recordName . '_chunk_200_299' => array_fill(200, 45, 'test'), ); - $this->assertSame($expected, $this->chunk->moveArchiveBlobsIntoChunks($array)); + $this->assertSame($expected, $this->chunk->moveArchiveBlobsIntoChunks($this->recordName, $array)); } /** diff --git a/tests/PHPUnit/Unit/ArchiveProcessorTest.php b/tests/PHPUnit/Unit/ArchiveProcessorTest.php deleted file mode 100644 index 1f092e3d03..0000000000 --- a/tests/PHPUnit/Unit/ArchiveProcessorTest.php +++ /dev/null @@ -1,129 +0,0 @@ -<?php -/** - * Piwik - free/libre analytics platform - * - * @link http://piwik.org - * @license http://www.gnu.org/licenses/gpl-3.0.html GPL v3 or later - */ - -namespace Piwik\Tests\Unit; - -use Piwik\Archive\Chunk; -use Piwik\ArchiveProcessor; -use Piwik\ArchiveProcessor\Parameters; -use Piwik\DataTable; -use Piwik\Segment; -use Piwik\Tests\Framework\Mock\Site; -use Piwik\Tests\Framework\TestCase\UnitTestCase; -use Piwik\Period\Factory as PeriodFactory; - -/** - * @group ArchiveProcessorTest - * @group ArchiveProcessor - * @group Archive - * @group Core - */ -class ArchiveProcessorTest extends UnitTestCase -{ - - public function test_insertBlobRecord_NoBlobsGiven() - { - $this->assertInsertBlobRecordPassesBlobsToArchiveWriter(array(), array()); - } - - public function test_insertBlobRecord_OnlyRootTableGiven_ShouldNotMoveRootTableIntoAChunk() - { - $blobs = array(0 => $this->getSerializedBlob()); - $this->assertInsertBlobRecordPassesBlobsToArchiveWriter($blobs, $blobs); - } - - public function test_insertBlobRecord_RootAndSubTablesGiven_OnlyAfewSubtables() - { - $blobs = $this->generateBlobs(0, 45); - - $expectedBlobs = array( - 0 => $this->getSerializedBlob('_0'), - 'chunk_0_99' => serialize($this->generateBlobs(1, 44)), // does not start with zero as zero is root table - ); - - $this->assertInsertBlobRecordPassesBlobsToArchiveWriter($expectedBlobs, $blobs); - } - - public function test_insertBlobRecord_RootAndSubTablesGiven_ShouldOnlySplitSubtablesIntoAChunk() - { - $blobs = $this->generateBlobs(0, 1145); - - $expectedBlobs = array( - 0 => $this->getSerializedBlob('_0'), - 'chunk_0_99' => serialize($this->generateBlobs(1, Chunk::NUM_TABLES_IN_CHUNK - 1)), // does not start with zero as zero is root table - 'chunk_100_199' => serialize($this->generateBlobs(100, Chunk::NUM_TABLES_IN_CHUNK)), - 'chunk_200_299' => serialize($this->generateBlobs(200, Chunk::NUM_TABLES_IN_CHUNK)), - 'chunk_300_399' => serialize($this->generateBlobs(300, Chunk::NUM_TABLES_IN_CHUNK)), - 'chunk_400_499' => serialize($this->generateBlobs(400, Chunk::NUM_TABLES_IN_CHUNK)), - 'chunk_500_599' => serialize($this->generateBlobs(500, Chunk::NUM_TABLES_IN_CHUNK)), - 'chunk_600_699' => serialize($this->generateBlobs(600, Chunk::NUM_TABLES_IN_CHUNK)), - 'chunk_700_799' => serialize($this->generateBlobs(700, Chunk::NUM_TABLES_IN_CHUNK)), - 'chunk_800_899' => serialize($this->generateBlobs(800, Chunk::NUM_TABLES_IN_CHUNK)), - 'chunk_900_999' => serialize($this->generateBlobs(900, Chunk::NUM_TABLES_IN_CHUNK)), - 'chunk_1000_1099' => serialize($this->generateBlobs(1000, Chunk::NUM_TABLES_IN_CHUNK)), - 'chunk_1100_1199' => serialize($this->generateBlobs(1100, 45)), - ); - - $this->assertInsertBlobRecordPassesBlobsToArchiveWriter($expectedBlobs, $blobs); - } - - public function test_insertBlobRecord_ShouldBeAbleToHandleAString() - { - $serialized = $this->getSerializedBlob(); - - $this->assertInsertBlobRecordPassesBlobsToArchiveWriter($serialized, $serialized); - } - - private function generateBlobs($startIndex, $numberOfEntries) - { - $blobs = array(); - - for ($i = 0; $i < $numberOfEntries; $i++) { - $subtableId = $startIndex + $i; - // we need to append something to make sure it actually moves the correct blob into the correct chunk - $blobs[$subtableId] = $this->getSerializedBlob('_'. $subtableId); - } - - return $blobs; - } - - private function getSerializedBlob($appendix = '') - { - return 'a:1:{i:0;a:3:{i:0;a:0:{}i:1;a:0:{}i:3;N;}}' . $appendix; - } - - private function assertInsertBlobRecordPassesBlobsToArchiveWriter($expectedBlobs, $blobs) - { - $recordName = 'Actions_Action_url'; - - $writer = $this->getMock('Piwik\DataAccess\ArchiveWriter', array('insertBlobRecord'), array(), '', false); - $writer->expects($this->once()) - ->method('insertBlobRecord') - ->with($recordName, $expectedBlobs); - - $processor = $this->createProcessor($writer); - $processor->insertBlobRecord($recordName, $blobs); - } - - private function createArchiveProcessorParamaters() - { - $oPeriod = PeriodFactory::makePeriodFromQueryParams('UTC', 'day', '2015-01-01'); - - $segment = new Segment(false, array(1)); - $params = new Parameters(new Site(1), $oPeriod, $segment); - - return $params; - } - - private function createProcessor($writer) - { - $params = $this->createArchiveProcessorParamaters(); - - return new ArchiveProcessor($params, $writer); - } -}
\ No newline at end of file diff --git a/tests/PHPUnit/Unit/DataAccess/ArchiveWriterTest.php b/tests/PHPUnit/Unit/DataAccess/ArchiveWriterTest.php index edc77e1ba9..292dd7682a 100644 --- a/tests/PHPUnit/Unit/DataAccess/ArchiveWriterTest.php +++ b/tests/PHPUnit/Unit/DataAccess/ArchiveWriterTest.php @@ -8,6 +8,7 @@ namespace Piwik\Tests\Unit; +use Piwik\Archive\Chunk; use Piwik\DataAccess\ArchiveWriter; use Piwik\DataTable; use Piwik\Segment; @@ -30,39 +31,44 @@ class ArchiveWriterTest extends UnitTestCase $this->assertInsertBlobRecordInsertedRecordsInBulk(array(), array()); } - public function test_insertBlobRecord_ShouldAppendTheRecordNameToSubtables() + public function test_insertBlobRecord_OnlyRootTableGiven_ShouldNotMoveRootTableIntoAChunk() { - $blobs = array( - 0 => $this->getSerializedBlob('_root'), - 1 => $this->getSerializedBlob('subtable1'), - 4 => $this->getSerializedBlob('subtable4'), - 5 => $this->getSerializedBlob('subtable5') - ); + $blobs = array(0 => $this->getSerializedBlob()); + $expected = array(array($this->recordName, $this->getSerializedBlob())); + + $this->assertInsertBlobRecordInsertedRecordsInBulk($expected, $blobs); + } + + public function test_insertBlobRecord_RootAndSubTablesGiven_OnlyAfewSubtables() + { + $blobs = $this->generateBlobs(0, 45); $expectedBlobs = array( - array($this->recordName , $this->getSerializedBlob('_root')), - array($this->recordName . '_1', $this->getSerializedBlob('subtable1')), - array($this->recordName . '_4', $this->getSerializedBlob('subtable4')), - array($this->recordName . '_5', $this->getSerializedBlob('subtable5')) + array($this->recordName, $this->getSerializedBlob('_0')), + array($this->recordName . '_chunk_0_99', serialize($this->generateBlobs(1, 44))) ); $this->assertInsertBlobRecordInsertedRecordsInBulk($expectedBlobs, $blobs); } - public function test_insertBlobRecord_ShouldAppendTheRecordNameToChunks() + public function test_insertBlobRecord_RootAndSubTablesGiven_ShouldOnlySplitSubtablesIntoAChunk() { - $blobs = array( - 0 => $this->getSerializedBlob('_root'), - 'chunk_0_99' => $this->getSerializedBlob('chunk0'), - 'chunk_100_199' => $this->getSerializedBlob('chunk1'), - 'chunk_200_299' => $this->getSerializedBlob('chunk2') - ); + $blobs = $this->generateBlobs(0, 1145); $expectedBlobs = array( - array($this->recordName , $this->getSerializedBlob('_root')), - array($this->recordName . '_chunk_0_99', $this->getSerializedBlob('chunk0')), - array($this->recordName . '_chunk_100_199', $this->getSerializedBlob('chunk1')), - array($this->recordName . '_chunk_200_299', $this->getSerializedBlob('chunk2')) + array($this->recordName, $this->getSerializedBlob('_0')), + array($this->recordName . '_chunk_0_99' , serialize($this->generateBlobs(1, Chunk::NUM_TABLES_IN_CHUNK - 1))), // does not start with zero as zero is root table + array($this->recordName . '_chunk_100_199' , serialize($this->generateBlobs(100, Chunk::NUM_TABLES_IN_CHUNK))), + array($this->recordName . '_chunk_200_299' , serialize($this->generateBlobs(200, Chunk::NUM_TABLES_IN_CHUNK))), + array($this->recordName . '_chunk_300_399' , serialize($this->generateBlobs(300, Chunk::NUM_TABLES_IN_CHUNK))), + array($this->recordName . '_chunk_400_499' , serialize($this->generateBlobs(400, Chunk::NUM_TABLES_IN_CHUNK))), + array($this->recordName . '_chunk_500_599' , serialize($this->generateBlobs(500, Chunk::NUM_TABLES_IN_CHUNK))), + array($this->recordName . '_chunk_600_699' , serialize($this->generateBlobs(600, Chunk::NUM_TABLES_IN_CHUNK))), + array($this->recordName . '_chunk_700_799' , serialize($this->generateBlobs(700, Chunk::NUM_TABLES_IN_CHUNK))), + array($this->recordName . '_chunk_800_899' , serialize($this->generateBlobs(800, Chunk::NUM_TABLES_IN_CHUNK))), + array($this->recordName . '_chunk_900_999' , serialize($this->generateBlobs(900, Chunk::NUM_TABLES_IN_CHUNK))), + array($this->recordName . '_chunk_1000_1099', serialize($this->generateBlobs(1000, Chunk::NUM_TABLES_IN_CHUNK))), + array($this->recordName . '_chunk_1100_1199', serialize($this->generateBlobs(1100, 45))), ); $this->assertInsertBlobRecordInsertedRecordsInBulk($expectedBlobs, $blobs); @@ -75,6 +81,19 @@ class ArchiveWriterTest extends UnitTestCase $this->assertInsertBlobRecordInsertedASingleRecord($blob, $blob); } + private function generateBlobs($startIndex, $numberOfEntries) + { + $blobs = array(); + + for ($i = 0; $i < $numberOfEntries; $i++) { + $subtableId = $startIndex + $i; + // we need to append something to make sure it actually moves the correct blob into the correct chunk + $blobs[$subtableId] = $this->getSerializedBlob('_'. $subtableId); + } + + return $blobs; + } + private function getSerializedBlob($appendix = '') { return 'a:1:{i:0;a:3:{i:0;a:0:{}i:1;a:0:{}i:3;N;}}' . $appendix; @@ -83,7 +102,7 @@ class ArchiveWriterTest extends UnitTestCase private function assertInsertBlobRecordInsertedRecordsInBulk($expectedBlobs, $blobs) { $writer = $this->getMock('Piwik\DataAccess\ArchiveWriter', array('insertBulkRecords', 'compress'), array(), '', false); - $writer->expects($this->exactly(count($blobs))) + $writer->expects($this->exactly(count($expectedBlobs))) ->method('compress') ->will($this->returnArgument(0)); $writer->expects($this->once()) |