From 8ab3a1f5bf39bb6fa148ecf605d11583b37c82b5 Mon Sep 17 00:00:00 2001 From: Daria Volvenkova Date: Fri, 5 Oct 2018 13:44:20 +0300 Subject: Local ads features processing for geofencing. --- map/local_ads_manager.cpp | 516 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 388 insertions(+), 128 deletions(-) (limited to 'map/local_ads_manager.cpp') diff --git a/map/local_ads_manager.cpp b/map/local_ads_manager.cpp index 0ac3b9c06e..fb2f56e034 100644 --- a/map/local_ads_manager.cpp +++ b/map/local_ads_manager.cpp @@ -22,6 +22,7 @@ #include "coding/file_name_utils.hpp" #include "coding/multilang_utf8_string.hpp" +#include "coding/point_to_integer.hpp" #include "coding/url_encode.hpp" #include "base/url_helpers.hpp" @@ -42,6 +43,7 @@ std::string const kServerUrl = LOCAL_ADS_SERVER_URL; std::string const kCampaignPageUrl = LOCAL_ADS_COMPANY_PAGE_URL; std::string const kCampaignFile = "local_ads_campaigns.dat"; +std::string const kCampaignFeaturesFile = "local_ads_features.dat"; std::string const kLocalAdsSymbolsFile = "local_ads_symbols.txt"; auto constexpr kWWanUpdateTimeout = std::chrono::hours(12); uint8_t constexpr kRequestMinZoomLevel = 12; @@ -50,6 +52,11 @@ auto constexpr kMaxDownloadingAttempts = 5; auto constexpr kHiddenFeaturePriority = 1; +double constexpr kMinCheckDistanceInMeters = 10.0; +double constexpr kMinSearchRadiusInMeters = 20.0; +uint32_t constexpr kMaxHitCount = 3; +auto const kMinCheckInterval = std::chrono::minutes(1); + void SerializeCampaign(FileWriter & writer, std::string const & countryName, LocalAdsManager::Timestamp const & ts, std::vector const & rawData) @@ -69,7 +76,7 @@ void DeserializeCampaign(ReaderSource & src, std::string & countryNa std::string GetPath(std::string const & fileName) { - return base::JoinFoldersToPath(GetPlatform().WritableDir(), fileName); + return base::JoinFoldersToPath(GetPlatform().SettingsDir(), fileName); } std::string MakeCampaignDownloadingURL(MwmSet::MwmId const & mwmId) @@ -133,149 +140,97 @@ std::string GetCustomIcon(FeatureType & featureType) return {}; } -using CampaignData = std::map>; - -CampaignData ParseCampaign(std::vector const & rawData, MwmSet::MwmId const & mwmId, - LocalAdsManager::Timestamp timestamp, - LocalAdsManager::GetFeatureByIdFn const & getFeatureByIdFn) +std::string MakeCampaignPageURL(FeatureID const & featureId) { - ASSERT(getFeatureByIdFn != nullptr, ()); - CampaignData data; - auto campaigns = local_ads::Deserialize(rawData); - for (local_ads::Campaign const & campaign : campaigns) - { - FeatureID featureId(mwmId, campaign.m_featureId); - if (!featureId.IsValid()) - continue; + if (kCampaignPageUrl.empty() || !featureId.m_mwmId.IsAlive()) + return {}; - if (campaign.m_priority == kHiddenFeaturePriority) - { - data.insert(std::make_pair(featureId, nullptr)); - continue; - } + std::ostringstream ss; + ss << kCampaignPageUrl << "/" << featureId.m_mwmId.GetInfo()->GetVersion() << "/" + << UrlEncode(featureId.m_mwmId.GetInfo()->GetCountryName()) << "/" << featureId.m_index; - std::string iconName = campaign.GetIconName(); - auto const expiration = timestamp + std::chrono::hours(24 * campaign.m_daysBeforeExpired); - if (iconName.empty() || local_ads::Clock::now() > expiration) + url::Params params; + params.reserve(kMarketingParameters.size()); + for (auto const & key : kMarketingParameters) + { + string value; + if (!marketing::Settings::Get(key, value)) continue; - FeatureType featureType; - if (getFeatureByIdFn(featureId, featureType)) - { - auto const customIcon = GetCustomIcon(featureType); - if (!customIcon.empty()) - iconName = customIcon; - } - - auto markData = std::make_shared(); - markData->m_symbolName = iconName; - markData->m_minZoomLevel = campaign.m_minZoomLevel; - data.insert(std::make_pair(featureId, std::move(markData))); + params.push_back({key, value}); } - return data; + return url::Make(ss.str(), params); } +} // namespace -df::CustomFeatures ReadCampaignFeatures(LocalAdsManager::ReadFeaturesFn const & reader, - CampaignData & campaignData) +namespace features_cache +{ +enum class Version : uint8_t { - ASSERT(reader != nullptr, ()); + V0 = 0, + Latest = V0 +}; - std::vector features; - features.reserve(campaignData.size()); - df::CustomFeatures customFeatures; - for (auto const & data : campaignData) - { - if (data.second) - features.push_back(data.first); - customFeatures.insert(std::make_pair(data.first, data.second != nullptr)); - } +struct PackedCampaignFeature +{ + PackedCampaignFeature() = default; + PackedCampaignFeature(uint32_t featureIndex, int64_t mercator) + : m_featureIndex(featureIndex) + , m_mercator(mercator) + {} - auto const deviceLang = StringUtf8Multilang::GetLangIndex(languages::GetCurrentNorm()); - reader( - [&campaignData, deviceLang](FeatureType & ft) { - auto it = campaignData.find(ft.GetID()); - CHECK(it != campaignData.end(), ()); - CHECK(it->second != nullptr, ()); - it->second->m_position = feature::GetCenter(ft, scales::GetUpperScale()); - ft.GetPreferredNames(true /* allowTranslit */, deviceLang, it->second->m_mainText, - it->second->m_auxText); - }, - features); + uint32_t m_featureIndex = 0; + int64_t m_mercator = 0; +}; - return customFeatures; +void SerializeVersion(FileWriter & writer, Version version) +{ + WriteToSink(writer, static_cast(version)); } -void CreateLocalAdsMarks(BookmarkManager * bmManager, CampaignData && campaignData) +void SerializeMwmData(FileWriter & writer, std::string const & countryId, int64_t mwmVersion) { - if (bmManager == nullptr) - return; - - // Here we copy campaign data, because we can create user marks only from UI thread. - GetPlatform().RunTask(Platform::Thread::Gui, [bmManager, campaignData = std::move(campaignData)]() - { - auto editSession = bmManager->GetEditSession(); - for (auto const & data : campaignData) - { - if (data.second == nullptr) - continue; - - auto * mark = editSession.CreateUserMark(data.second->m_position); - mark->SetData(LocalAdsMarkData(*data.second)); - mark->SetFeatureId(data.first); - } - }); + local_ads::WriteCountryName(writer, countryId); + local_ads::WriteZigZag(writer, mwmVersion); } -void DeleteLocalAdsMarks(BookmarkManager * bmManager, MwmSet::MwmId const & mwmId) +void SerializePackedFeatures(FileWriter & writer, std::vector const & packedFeatures) { - if (bmManager == nullptr) - return; - - GetPlatform().RunTask(Platform::Thread::Gui, [bmManager, mwmId]() + auto const featuresCount = static_cast(packedFeatures.size()); + WriteToSink(writer, featuresCount); + for (auto const & data : packedFeatures) { - bmManager->GetEditSession().DeleteUserMarks(UserMark::Type::LOCAL_ADS, - [&mwmId](LocalAdsMark const * mark) - { - return mark->GetFeatureID().m_mwmId == mwmId; - }); - }); + WriteToSink(writer, data.m_featureIndex); + local_ads::WriteZigZag(writer, data.m_mercator); + } } -void DeleteAllLocalAdsMarks(BookmarkManager * bmManager) +Version DeserializeVersion(ReaderSource & src) { - if (bmManager == nullptr) - return; - - GetPlatform().RunTask(Platform::Thread::Gui, [bmManager]() - { - bmManager->GetEditSession().ClearGroup(UserMark::Type::LOCAL_ADS); - }); + return static_cast(ReadPrimitiveFromSource(src)); } -std::string MakeCampaignPageURL(FeatureID const & featureId) +void DeserializeMwmData(ReaderSource & src, std::string & countryId, int64_t & mwmVersion) { - if (kCampaignPageUrl.empty() || !featureId.m_mwmId.IsAlive()) - return {}; - - std::ostringstream ss; - ss << kCampaignPageUrl << "/" << featureId.m_mwmId.GetInfo()->GetVersion() << "/" - << UrlEncode(featureId.m_mwmId.GetInfo()->GetCountryName()) << "/" << featureId.m_index; + countryId = local_ads::ReadCountryName(src); + mwmVersion = local_ads::ReadZigZag(src); +} - url::Params params; - params.reserve(kMarketingParameters.size()); - for (auto const & key : kMarketingParameters) +void DeserializePackedFeatures(ReaderSource & src, std::vector & packedFeatures) +{ + auto const featuresCount = ReadPrimitiveFromSource(src); + packedFeatures.clear(); + packedFeatures.reserve(static_cast(featuresCount)); + for (uint32_t i = 0; i < featuresCount; ++i) { - string value; - if (!marketing::Settings::Get(key, value)) - continue; - - params.push_back({key, value}); + PackedCampaignFeature data; + data.m_featureIndex = ReadPrimitiveFromSource(src); + data.m_mercator = local_ads::ReadZigZag(src); + packedFeatures.push_back(data); } - - return url::Make(ss.str(), params); } -} // namespace +} // namespace features_cache LocalAdsManager::LocalAdsManager(GetMwmsByRectFn && getMwmsByRectFn, GetMwmIdByNameFn && getMwmIdByName, @@ -342,6 +297,11 @@ void LocalAdsManager::UpdateViewport(ScreenBase const & screen) if (mwms.empty()) return; + if (m_lastMwms == mwms) + return; + + m_lastMwms = mwms; + // Request local ads campaigns. GetPlatform().RunTask(Platform::Thread::File, [this, mwms = std::move(mwms)]() { @@ -461,7 +421,6 @@ void LocalAdsManager::ProcessRequests(std::set && requests) { GetPlatform().RunTask(Platform::Thread::Network, [this, requests = std::move(requests)] { - std::string const campaignFile = GetPath(kCampaignFile); for (auto const & request : requests) { auto const & mwm = request.first; @@ -486,8 +445,8 @@ void LocalAdsManager::ProcessRequests(std::set && requests) auto campaignData = ParseCampaign(info.m_data, mwm, info.m_created, m_getFeatureByIdFn); if (!campaignData.empty()) { - UpdateFeaturesCache(ReadCampaignFeatures(m_readFeaturesFn, campaignData)); - CreateLocalAdsMarks(m_bmManager, std::move(campaignData)); + UpdateFeaturesCache(ReadCampaignFeatures(campaignData)); + CreateLocalAdsMarks(std::move(campaignData)); } } @@ -503,17 +462,22 @@ void LocalAdsManager::ProcessRequests(std::set && requests) ClearLocalAdsForMwm(mwm); } } - + + auto const campaignFile = GetPath(kCampaignFile); + auto const featureFile = GetPath(kCampaignFeaturesFile); + // Save data persistently. - GetPlatform().RunTask(Platform::Thread::File, [this, campaignFile] + GetPlatform().RunTask(Platform::Thread::File, [this, campaignFile, featureFile] { WriteCampaignFile(campaignFile); + WriteFeaturesFile(featureFile); }); }); } void LocalAdsManager::OnDownloadCountry(std::string const & countryName) { + m_lastMwms.clear(); GetPlatform().RunTask(Platform::Thread::File, [this, countryName] { std::lock_guard lock(m_campaignsMutex); @@ -524,6 +488,7 @@ void LocalAdsManager::OnDownloadCountry(std::string const & countryName) void LocalAdsManager::OnMwmDeregistered(platform::LocalCountryFile const & countryFile) { + m_lastMwms.clear(); MwmSet::MwmId mwmId; { std::lock_guard lock(m_featuresCacheMutex); @@ -587,11 +552,45 @@ void LocalAdsManager::WriteCampaignFile(std::string const & campaignFile) } } +void LocalAdsManager::WriteFeaturesFile(std::string const & featuresFile) +{ + std::lock_guard lock(m_featuresCacheMutex); + try + { + FileWriter writer(featuresFile); + features_cache::SerializeVersion(writer, features_cache::Version::V0); + + MwmSet::MwmId lastMwm; + std::vector packedData; + for (auto const & entry : m_featuresCache) + { + FeatureID const & fid = entry.first; + if (lastMwm != fid.m_mwmId && !packedData.empty()) + { + features_cache::SerializeMwmData(writer, lastMwm.GetInfo()->GetCountryName(), lastMwm.GetInfo()->GetVersion()); + features_cache::SerializePackedFeatures(writer, packedData); + packedData.clear(); + } + lastMwm = fid.m_mwmId; + packedData.emplace_back(fid.m_index, PointToInt64Obsolete(entry.second.m_position, POINT_COORD_BITS)); + } + if (!packedData.empty()) + { + features_cache::SerializeMwmData(writer, lastMwm.GetInfo()->GetCountryName(), lastMwm.GetInfo()->GetVersion()); + features_cache::SerializePackedFeatures(writer, packedData); + } + } + catch (RootException const & ex) + { + LOG(LWARNING, ("Error writing file:", featuresFile, ex.Msg())); + } +} + void LocalAdsManager::Invalidate() { GetPlatform().RunTask(Platform::Thread::File, [this] { - DeleteAllLocalAdsMarks(m_bmManager); + DeleteAllLocalAdsMarks(); m_drapeEngine.SafeCall(&df::DrapeEngine::RemoveAllCustomFeatures); CampaignData campaignData; @@ -604,21 +603,142 @@ void LocalAdsManager::Invalidate() campaignData.insert(data.begin(), data.end()); } } - UpdateFeaturesCache(ReadCampaignFeatures(m_readFeaturesFn, campaignData)); - CreateLocalAdsMarks(m_bmManager, std::move(campaignData)); + UpdateFeaturesCache(ReadCampaignFeatures(campaignData)); + CreateLocalAdsMarks(std::move(campaignData)); + }); +} + +LocalAdsManager::CampaignData LocalAdsManager::ParseCampaign(std::vector const & rawData, + MwmSet::MwmId const & mwmId, + Timestamp timestamp, + GetFeatureByIdFn const & getFeatureByIdFn) const +{ + ASSERT(getFeatureByIdFn != nullptr, ()); + CampaignData data; + auto campaigns = local_ads::Deserialize(rawData); + for (local_ads::Campaign const & campaign : campaigns) + { + FeatureID featureId(mwmId, campaign.m_featureId); + if (!featureId.IsValid()) + continue; + + if (campaign.m_priority == kHiddenFeaturePriority) + { + data.insert(std::make_pair(featureId, nullptr)); + continue; + } + + std::string iconName = campaign.GetIconName(); + auto const expiration = timestamp + std::chrono::hours(24 * campaign.m_daysBeforeExpired); + if (iconName.empty() || local_ads::Clock::now() > expiration) + continue; + + FeatureType featureType; + if (getFeatureByIdFn(featureId, featureType)) + { + auto const customIcon = GetCustomIcon(featureType); + if (!customIcon.empty()) + iconName = customIcon; + } + + auto markData = std::make_shared(); + markData->m_symbolName = iconName; + markData->m_minZoomLevel = campaign.m_minZoomLevel; + data.insert(std::make_pair(featureId, std::move(markData))); + } + + return data; +} + +void LocalAdsManager::CreateLocalAdsMarks(CampaignData && campaignData) +{ + if (m_bmManager == nullptr) + return; + + // Here we copy campaign data, because we can create user marks only from UI thread. + GetPlatform().RunTask(Platform::Thread::Gui, [this, campaignData = std::move(campaignData)]() + { + auto editSession = m_bmManager->GetEditSession(); + for (auto const & data : campaignData) + { + if (data.second == nullptr) + continue; + + auto * mark = editSession.CreateUserMark(data.second->m_position); + mark->SetData(LocalAdsMarkData(*data.second)); + mark->SetFeatureId(data.first); + } }); } -void LocalAdsManager::UpdateFeaturesCache(df::CustomFeatures && features) +void LocalAdsManager::DeleteLocalAdsMarks(MwmSet::MwmId const & mwmId) +{ + if (m_bmManager == nullptr) + return; + + GetPlatform().RunTask(Platform::Thread::Gui, [this, mwmId]() + { + m_bmManager->GetEditSession().DeleteUserMarks(UserMark::Type::LOCAL_ADS, + [&mwmId](LocalAdsMark const * mark) + { + return mark->GetFeatureID().m_mwmId == mwmId; + }); + }); +} + +void LocalAdsManager::DeleteAllLocalAdsMarks() +{ + if (m_bmManager == nullptr) + return; + + GetPlatform().RunTask(Platform::Thread::Gui, [this]() + { + m_bmManager->GetEditSession().ClearGroup(UserMark::Type::LOCAL_ADS); + }); +} + +LocalAdsManager::FeaturesCache LocalAdsManager::ReadCampaignFeatures(CampaignData & campaignData) const +{ + ASSERT(m_readFeaturesFn != nullptr, ()); + + LocalAdsManager::FeaturesCache cache; + + std::vector features; + features.reserve(campaignData.size()); + for (auto const & data : campaignData) + features.push_back(data.first); + + auto const deviceLang = StringUtf8Multilang::GetLangIndex(languages::GetCurrentNorm()); + m_readFeaturesFn( + [&campaignData, &cache, deviceLang](FeatureType & ft) { + auto it = campaignData.find(ft.GetID()); + CHECK(it != campaignData.end(), ()); + + LocalAdsManager::CacheEntry entry; + entry.m_position = feature::GetCenter(ft, scales::GetUpperScale()); + entry.m_isCustom = it->second != nullptr; + if (it->second != nullptr) + { + it->second->m_position = entry.m_position; + ft.GetPreferredNames(true /* allowTranslit */, deviceLang, it->second->m_mainText, it->second->m_auxText); + } + cache.insert(std::make_pair(it->first, entry)); + }, + features); + return cache; +} + +void LocalAdsManager::UpdateFeaturesCache(FeaturesCache && features) { - df::CustomFeatures featuresCache; + df::CustomFeatures customFeatures; { std::lock_guard lock(m_featuresCacheMutex); if (!features.empty()) m_featuresCache.insert(features.begin(), features.end()); - featuresCache = m_featuresCache; + for (auto const & entry : m_featuresCache) + customFeatures.insert(std::make_pair(entry.first, entry.second.m_isCustom)); } - m_drapeEngine.SafeCall(&df::DrapeEngine::SetCustomFeatures, std::move(featuresCache)); + m_drapeEngine.SafeCall(&df::DrapeEngine::SetCustomFeatures, std::move(customFeatures)); } void LocalAdsManager::ClearLocalAdsForMwm(MwmSet::MwmId const & mwmId) @@ -639,7 +759,7 @@ void LocalAdsManager::ClearLocalAdsForMwm(MwmSet::MwmId const & mwmId) m_drapeEngine.SafeCall(&df::DrapeEngine::RemoveCustomFeatures, mwmId); // Delete marks. - DeleteLocalAdsMarks(m_bmManager, mwmId); + DeleteLocalAdsMarks(mwmId); } bool LocalAdsManager::Contains(FeatureID const & featureId) const @@ -668,6 +788,7 @@ void LocalAdsManager::OnSubscriptionChanged(SubscriptionType type, bool isActive return; m_isEnabled = enabled; + m_lastMwms.clear(); if (enabled) { if (!m_isStarted) @@ -695,7 +816,7 @@ void LocalAdsManager::OnSubscriptionChanged(SubscriptionType type, bool isActive } // Clear all graphics. - DeleteAllLocalAdsMarks(m_bmManager); + DeleteAllLocalAdsMarks(); m_drapeEngine.SafeCall(&df::DrapeEngine::RemoveAllCustomFeatures); }); @@ -704,8 +825,147 @@ void LocalAdsManager::OnSubscriptionChanged(SubscriptionType type, bool isActive } } +void LocalAdsManager::OnLocationUpdate(location::GpsInfo const & info, int zoomLevel) +{ + if (!m_isEnabled) + return; + + if (std::chrono::steady_clock::now() < (m_lastCheckTime + kMinCheckInterval)) + return; + + auto const pt = MercatorBounds::FromLatLon(info.m_latitude, info.m_longitude); + + if ((m_foundFeatureHitCount == 0 || m_foundFeatureHitCount == kMaxHitCount) && + MercatorBounds::DistanceOnEarth(m_lastCheckedPos, pt) < kMinCheckDistanceInMeters) + { + return; + } + + auto const radius = std::max(info.m_horizontalAccuracy, kMinSearchRadiusInMeters); + auto searchRect = MercatorBounds::RectByCenterXYAndSizeInMeters(pt, radius); + + FeatureID fid; + { + std::lock_guard lock(m_featuresCacheMutex); + double minDist = numeric_limits::max(); + for (auto const & pair : m_featuresCache) + { + auto const & pos = pair.second.m_position; + if (!searchRect.IsPointInside(pos)) + continue; + auto const dist = MercatorBounds::DistanceOnEarth(pos, pt); + if (dist < radius && dist < minDist) + { + minDist = dist; + fid = pair.first; + } + } + } + + m_lastCheckTime = std::chrono::steady_clock::now(); + m_lastCheckedPos = pt; + + if (fid.IsValid()) + { + m_statistics.RegisterEvent(local_ads::Event(local_ads::EventType::Visit, fid.m_mwmId.GetInfo()->GetVersion(), + fid.m_mwmId.GetInfo()->GetCountryName(), fid.m_index, + static_cast(zoomLevel), local_ads::Clock::now(), + info.m_latitude, info.m_longitude, + static_cast(info.m_horizontalAccuracy))); + if (m_lastFoundFeature != fid) + m_foundFeatureHitCount = 1; + else if (m_foundFeatureHitCount < kMaxHitCount) + ++m_foundFeatureHitCount; + } + else + { + m_foundFeatureHitCount = 0; + } + m_lastFoundFeature = fid; +} + bool LocalAdsManager::BackoffStats::CanRetry() const { return !m_fileIsAbsent && m_attemptsCount < kMaxDownloadingAttempts && std::chrono::steady_clock::now() > (m_lastDownloading + m_currentTimeout); } + +namespace lightweight +{ +void LocalAdsFeaturesReader::ReadCampaignFeaturesFile() +{ + m_features.clear(); + std::string const featuresFile = GetPath(kCampaignFeaturesFile); + try + { + FileReader reader(featuresFile); + ReaderSource src(reader); + + auto const version = features_cache::DeserializeVersion(src); + if (version != features_cache::Version::V0) + return; + + std::string countryId; + int64_t mwmVersion; + std::vector packedData; + while (src.Size() > 0) + { + features_cache::DeserializeMwmData(src, countryId, mwmVersion); + features_cache::DeserializePackedFeatures(src, packedData); + + for (auto const & data : packedData) + { + auto const pos = Int64ToPointObsolete(data.m_mercator, POINT_COORD_BITS); + m_features.push_back(CampaignFeature(mwmVersion, countryId, data.m_featureIndex, + MercatorBounds::YToLat(pos.y), MercatorBounds::XToLon(pos.x))); + } + } + } + catch (Reader::Exception const & ex) + { + LOG(LWARNING, ("Error reading file:", featuresFile, ex.Msg())); + FileWriter::DeleteFileX(featuresFile); + } +} + +std::vector LocalAdsFeaturesReader::GetCampaignFeatures(double lat, double lon, double radiusInMeters, + uint32_t maxCount) +{ + if (!m_initialized) + { + ReadCampaignFeaturesFile(); + m_initialized = true; + } + + if (m_features.empty()) + return {}; + + auto const pt = MercatorBounds::FromLatLon(lat, lon); + auto searchRect = MercatorBounds::RectByCenterXYAndSizeInMeters(pt, radiusInMeters); + + std::multimap sortedFeatures; + for (auto const & f : m_features) + { + auto const pos = MercatorBounds::FromLatLon(f.m_lat, f.m_lon); + if (!searchRect.IsPointInside(pos)) + continue; + auto const dist = static_cast(MercatorBounds::DistanceOnEarth(pos, pt)); + sortedFeatures.insert(std::make_pair(dist, f)); + } + + std::vector filteredFeatures; + filteredFeatures.reserve(std::min(sortedFeatures.size(), static_cast(maxCount))); + for (auto const & pair : sortedFeatures) + { + filteredFeatures.push_back(pair.second); + if (filteredFeatures.size() == maxCount) + break; + } + return filteredFeatures; +} + +void Statistics::RegisterEvent(local_ads::Event && event) +{ + m_statistics.RegisterEventSync(std::move(event)); +} +} // namespace lightweight -- cgit v1.2.3