diff options
author | VladiMihaylenko <vxmihaylenko@gmail.com> | 2018-06-20 15:31:58 +0300 |
---|---|---|
committer | mpimenov <mpimenov@users.noreply.github.com> | 2018-06-22 17:36:17 +0300 |
commit | 673f9bdfdbe64608f2c66693d532519a2287ab9d (patch) | |
tree | 2b0259a3fd5f44aa60183d06686115e331eb2de9 /ugc | |
parent | cec8f91544707c624670b10eca6ecbc49d045dfc (diff) |
Added migration tables validation
Diffstat (limited to 'ugc')
-rw-r--r-- | ugc/ugc_tests/migration/generate_migration_files.cpp | 146 |
1 files changed, 120 insertions, 26 deletions
diff --git a/ugc/ugc_tests/migration/generate_migration_files.cpp b/ugc/ugc_tests/migration/generate_migration_files.cpp index 9fae971393..3693b92aa8 100644 --- a/ugc/ugc_tests/migration/generate_migration_files.cpp +++ b/ugc/ugc_tests/migration/generate_migration_files.cpp @@ -3,6 +3,7 @@ #include "coding/file_name_utils.hpp" #include "coding/file_reader.hpp" #include "coding/file_writer.hpp" +#include "coding/zlib.hpp" #include "indexer/classificator.hpp" #include "indexer/classificator_loader.hpp" @@ -19,6 +20,9 @@ using namespace std; namespace { +using Source = ReaderSource<FileReader>; +using MigrationTable = unordered_map<uint32_t, uint32_t>; + array<string, 11> const kVersions{{"171020", "171117", "171208", @@ -32,35 +36,62 @@ array<string, 11> const kVersions{{"171020", "180528"}}; string const kUGCMigrationDirName = "ugc_migration"; -string const kClassificatorFileName = "classificator.txt"; -string const kTypesFileName = "types.txt"; +string const kClassificatorFileName = "classificator.gz"; +string const kTypesFileName = "types.gz"; string const kBinFileExtension = ".bin"; + +string GetUGCDirPath() +{ + return my::JoinPath(GetPlatform().WritableDir(), kUGCMigrationDirName); +} + +void LoadClassificatorTypesForVersion(string const & version) +{ + auto const ugcDirPath = GetUGCDirPath(); + auto const folderPath = my::JoinPath(ugcDirPath, version); + auto const & p = GetPlatform(); + + using Inflate = coding::ZLib::Inflate; + + string classificator; + { + auto const r = p.GetReader(my::JoinPath(folderPath, kClassificatorFileName)); + string data; + r->ReadAsString(data); + Inflate inflate(Inflate::Format::GZip); + inflate(data.data(), data.size(), back_inserter(classificator)); + } + + string types; + { + auto const r = p.GetReader(my::JoinPath(folderPath, kTypesFileName)); + string data; + r->ReadAsString(data); + Inflate inflate(Inflate::Format::GZip); + inflate(data.data(), data.size(), back_inserter(types)); + } + + classificator::LoadTypes(classificator, types); +} + +void LoadTableForVersion(string const & version, MigrationTable & table) +{ + Source source(FileReader(my::JoinPath(GetUGCDirPath(), version + kBinFileExtension))); + ugc::DeserializerV0<Source> des(source); + des(table); +} } // namespace UNIT_TEST(UGC_GenerateMigrationFiles) { - auto & p = GetPlatform(); - auto const ugcDirPath = my::JoinPath(p.WritableDir(), kUGCMigrationDirName); + auto const ugcDirPath = GetUGCDirPath(); for (auto const & v : kVersions) { - auto const folderPath = my::JoinPath(ugcDirPath, v); - string classificator; - { - auto const r = p.GetReader(my::JoinPath(folderPath, kClassificatorFileName)); - r->ReadAsString(classificator); - } + LoadClassificatorTypesForVersion(v); + auto const & c = classif(); - string types; - { - auto const r = p.GetReader(my::JoinPath(folderPath, kTypesFileName)); - r->ReadAsString(types); - } - - classificator::LoadTypes(classificator, types); - Classificator const & c = classif(); - - unordered_map<uint32_t, uint32_t> mapping; - auto const parse = [&c, &mapping](ClassifObject const * obj, uint32_t type) + MigrationTable mapping; + auto const parse = [&c, &mapping](ClassifObject const *, uint32_t type) { if (c.IsTypeValid(type)) mapping.emplace(type, c.GetIndexForType(type)); @@ -76,13 +107,76 @@ UNIT_TEST(UGC_GenerateMigrationFiles) ser(mapping); } - unordered_map<uint32_t, uint32_t> res; + MigrationTable res; + LoadTableForVersion(v, res); + TEST_EQUAL(res, mapping, ()); + } +} + +UNIT_TEST(UGC_ValidateMigrationTables) +{ + auto const ugcDirPath = GetUGCDirPath(); + for (auto const & v : kVersions) + { + MigrationTable table; + LoadTableForVersion(v, table); + TEST(!table.empty(), ("Version", v)); + + LoadClassificatorTypesForVersion(v); + auto const & c = classif(); + auto const parse = [&c, &table](ClassifObject const *, uint32_t type) + { + if (c.IsTypeValid(type)) + TEST_EQUAL(table[type], c.GetIndexForType(type), ()); + }; + + c.ForEachTree(parse); + } +} + +UNIT_TEST(UGC_TypesFromDifferentVersions) +{ + auto constexpr size = kVersions.size(); + for (size_t i = 0; i < size; ++i) + { + auto const & v = kVersions[i]; + MigrationTable table; + LoadTableForVersion(v, table); + LoadClassificatorTypesForVersion(v); + unordered_map<string, uint32_t> namesToTypes; { - ReaderSource<FileReader> source(FileReader(filePath, true /* withExceptions */)); - ugc::DeserializerV0<ReaderSource<FileReader>> des(source); - des(res); + auto const & c = classif(); + auto const parse = [&c, &namesToTypes](ClassifObject const *, uint32_t type) + { + if (c.IsTypeValid(type)) + namesToTypes.emplace(c.GetReadableObjectName(type), type); + }; + + c.ForEachTree(parse); } - TEST_EQUAL(res, mapping, ()); + for (size_t j = 0; j < size; ++j) + { + if (i == j) + continue; + + auto const & otherV = kVersions[j]; + MigrationTable otherTable; + LoadTableForVersion(otherV, otherTable); + LoadClassificatorTypesForVersion(otherV); + auto const & c = classif(); + auto const parse = [&c, &namesToTypes, &otherTable, &table](ClassifObject const *, uint32_t otherType) { + if (c.IsTypeValid(otherType)) + { + auto const otherReadableName = c.GetReadableObjectName(otherType); + if (namesToTypes.find(otherReadableName) != namesToTypes.end()) + { + auto const type = namesToTypes[otherReadableName]; + TEST_EQUAL(otherTable[otherType], table[type], ()); + } + } + }; + c.ForEachTree(parse); + } } } |