Ping downloading urls

13 files changed, 183 insertions, 36 deletions

diff --git a/storage/CMakeLists.txt b/storage/CMakeLists.txt
index a38d7059d0..df402c24de 100644
--- a/storage/CMakeLists.txt
+++ b/storage/CMakeLists.txt
@@ -32,6 +32,8 @@ set(
   map_files_downloader.hpp
   queued_country.cpp
   queued_country.hpp
+  pinger.cpp
+  pinger.hpp
   routing_helpers.cpp
   routing_helpers.hpp
   storage.cpp
diff --git a/storage/http_map_files_downloader.cpp b/storage/http_map_files_downloader.cpp
index 2c03979276..1c8b11cfdb 100644
--- a/storage/http_map_files_downloader.cpp
+++ b/storage/http_map_files_downloader.cpp
@@ -31,12 +31,11 @@ HttpMapFilesDownloader::~HttpMapFilesDownloader()
   ASSERT_THREAD_CHECKER(m_checker, ());
 }
 
-void HttpMapFilesDownloader::GetServersList(int64_t const mapVersion, string const & mapFileName,
-                                            TServersListCallback const & callback)
+void HttpMapFilesDownloader::GetServersList(TServersListCallback const & callback)
 {
   ASSERT_THREAD_CHECKER(m_checker, ());
-  m_request.reset(downloader::HttpRequest::PostJson(
-      GetPlatform().MetaServerUrl(), strings::to_string(mapVersion) + '/' + mapFileName,
+  m_request.reset(downloader::HttpRequest::Get(
+      GetPlatform().MetaServerUrl(),
       bind(&HttpMapFilesDownloader::OnServersListDownloaded, this, callback, _1)));
 }
 
diff --git a/storage/http_map_files_downloader.hpp b/storage/http_map_files_downloader.hpp
index 8028f43796..fabc253517 100644
--- a/storage/http_map_files_downloader.hpp
+++ b/storage/http_map_files_downloader.hpp
@@ -17,7 +17,8 @@ public:
   virtual ~HttpMapFilesDownloader();
 
   // MapFilesDownloader overrides:
-  void GetServersList(int64_t const mapVersion, string const & mapFileName, TServersListCallback const & callback) override;
+  void GetServersList(TServersListCallback const & callback) override;
+
   void DownloadMapFile(vector<string> const & urls, string const & path, int64_t size,
                        TFileDownloadedCallback const & onDownloaded,
                        TDownloadingProgressCallback const & onProgress) override;
diff --git a/storage/map_files_downloader.hpp b/storage/map_files_downloader.hpp
index afa12a2e4f..1cd7b8b7b8 100644
--- a/storage/map_files_downloader.hpp
+++ b/storage/map_files_downloader.hpp
@@ -24,8 +24,7 @@ public:
 
   /// Asynchronously receives a list of all servers that can be asked
   /// for a map file and invokes callback on the original thread.
-  virtual void GetServersList(int64_t const mapVersion, string const & mapFileName,
-                              TServersListCallback const & callback) = 0;
+  virtual void GetServersList(TServersListCallback const & callback) = 0;
 
   /// Asynchronously downloads a map file, periodically invokes
   /// onProgress callback and finally invokes onDownloaded
diff --git a/storage/pinger.cpp b/storage/pinger.cpp
new file mode 100644
index 0000000000..4bd697624f
--- /dev/null
+++ b/storage/pinger.cpp
@@ -0,0 +1,75 @@
+#include "storage/pinger.hpp"
+
+#include "platform/http_client.hpp"
+#include "platform/preferred_languages.hpp"
+
+#include "base/assert.hpp"
+#include "base/logging.hpp"
+#include "base/stl_helpers.hpp"
+#include "base/worker_thread.hpp"
+
+#include "3party/Alohalytics/src/alohalytics.h"
+
+#include <sstream>
+#include <utility>
+
+using namespace std;
+
+namespace
+{
+auto constexpr kTimeoutInSeconds = 5.0;
+
+void DoPing(string const & url, size_t index, vector<string> & readyUrls)
+{
+  if (url.empty())
+  {
+    ASSERT(false, ("Metaserver returned an empty url."));
+    return;
+  }
+
+  platform::HttpClient request(url);
+  request.SetHttpMethod("HEAD");
+  request.SetTimeout(kTimeoutInSeconds);
+  if (request.RunHttpRequest() && !request.WasRedirected() && request.ErrorCode() == 200)
+  {
+    readyUrls[index] = url;
+  }
+  else
+  {
+    ostringstream ost;
+    ost << "Request to server " << url << " failed. Code = " << request.ErrorCode()
+        << ", redirection = " << request.WasRedirected();
+    LOG(LINFO, (ost.str()));
+  }
+}
+
+void SendStatistics(size_t serversLeft)
+{
+  alohalytics::Stats::Instance().LogEvent(
+      "Downloader_ServerList_check",
+      {{"lang", languages::GetCurrentNorm()}, {"servers", to_string(serversLeft)}});
+}
+}  // namespace
+
+namespace storage
+{
+// static
+void Pinger::Ping(vector<string> const & urls, Pinger::Pong const & pong)
+{
+  auto const size = urls.size();
+  CHECK_GREATER(size, 0, ());
+
+  vector<string> readyUrls(size);
+  {
+    base::WorkerThread t(size);
+    for (size_t i = 0; i < size; ++i)
+      t.Push([ url = urls[i], &readyUrls, i ] { DoPing(url, i, readyUrls); });
+
+    t.Shutdown(base::WorkerThread::Exit::ExecPending);
+  }
+
+  my::EraseIf(readyUrls, [](auto const & url) { return url.empty(); });
+  SendStatistics(readyUrls.size());
+  pong(move(readyUrls));
+}
+}  // namespace storage
diff --git a/storage/pinger.hpp b/storage/pinger.hpp
new file mode 100644
index 0000000000..6fd185455c
--- /dev/null
+++ b/storage/pinger.hpp
@@ -0,0 +1,16 @@
+#pragma once
+
+#include "platform/safe_callback.hpp"
+
+#include <string>
+#include <vector>
+
+namespace storage
+{
+class Pinger
+{
+public:
+  using Pong = platform::SafeCallback<void(std::vector<std::string> readyUrls)>;
+  static void Ping(std::vector<std::string> const & urls, Pong const & pong);
+};
+}  // namespace storage
diff --git a/storage/storage.cpp b/storage/storage.cpp
index 34083c4123..d0c7d33a79 100644
--- a/storage/storage.cpp
+++ b/storage/storage.cpp
@@ -99,7 +99,7 @@ MapFilesDownloader::TProgress Storage::GetOverallProgress(TCountriesVec const &
 
 Storage::Storage(string const & pathToCountriesFile /* = COUNTRIES_FILE */,
                  string const & dataDir /* = string() */)
-  : m_downloader(new HttpMapFilesDownloader())
+  : m_downloader(make_unique<HttpMapFilesDownloader>())
   , m_currentSlotId(0)
   , m_dataDir(dataDir)
   , m_downloadMapOnTheMap(nullptr)
@@ -108,6 +108,7 @@ Storage::Storage(string const & pathToCountriesFile /* = COUNTRIES_FILE */,
   SetLocale(languages::GetCurrentTwine());
   LoadCountriesFile(pathToCountriesFile, m_dataDir);
   CalMaxMwmSizeBytes();
+  LoadServerListForSession();
 }
 
 Storage::Storage(string const & referenceCountriesTxtJsonForTesting,
@@ -121,6 +122,7 @@ Storage::Storage(string const & referenceCountriesTxtJsonForTesting,
       LoadCountriesFromBuffer(referenceCountriesTxtJsonForTesting, m_countries, m_affiliations);
   CHECK_LESS_OR_EQUAL(0, m_currentVersion, ("Can't load test countries file"));
   CalMaxMwmSizeBytes();
+  LoadServerListForTesting();
 }
 
 void Storage::Init(TUpdateCallback const & didDownload, TDeleteCallback const & willDelete)
@@ -613,7 +615,6 @@ void Storage::DownloadNextCountryFromQueue()
 void Storage::DownloadNextFile(QueuedCountry const & country)
 {
   TCountryId const & countryId = country.GetCountryId();
-  CountryFile const & countryFile = GetCountryFile(countryId);
   auto const opt = country.GetCurrentFileOptions();
 
   string const readyFilePath = GetFileDownloadPath(countryId, opt);
@@ -641,9 +642,10 @@ void Storage::DownloadNextFile(QueuedCountry const & country)
     return;
   }
 
-  // send Country name for statistics
-  m_downloader->GetServersList(GetCurrentDataVersion(), countryFile.GetName(),
-                               bind(&Storage::OnServerListDownloaded, this, _1));
+  if (m_sessionServerList)
+    DoDownload();
+  else
+    SetDeferDownloading();
 }
 
 void Storage::DeleteFromDownloader(TCountryId const & countryId)
@@ -781,9 +783,10 @@ void Storage::ReportProgressForHierarchy(TCountryId const & countryId,
   ForEachAncestorExceptForTheRoot(countryId, calcProgress);
 }
 
-void Storage::OnServerListDownloaded(vector<string> const & urls)
+void Storage::DoDownload()
 {
   ASSERT_THREAD_CHECKER(m_threadChecker, ());
+  CHECK(m_sessionServerList, ());
 
   // Queue can be empty because countries were deleted from queue.
   if (m_queue.empty())
@@ -796,9 +799,7 @@ void Storage::OnServerListDownloaded(vector<string> const & urls)
     auto const status = m_diffManager.GetStatus();
     switch (status)
     {
-    case Status::Undefined:
-      m_deferredDownloads.push_back(urls);
-      return;
+    case Status::Undefined: SetDeferDownloading(); return;
     case Status::NotAvailable:
       queuedCountry.ResetToDefaultOptions();
       break;
@@ -810,7 +811,7 @@ void Storage::OnServerListDownloaded(vector<string> const & urls)
   }
 
   vector<string> const & downloadingUrls =
-      m_downloadingUrlsForTesting.empty() ? urls : m_downloadingUrlsForTesting;
+      m_downloadingUrlsForTesting.empty() ? *m_sessionServerList : m_downloadingUrlsForTesting;
   vector<string> fileUrls;
   fileUrls.reserve(downloadingUrls.size());
   for (string const & url : downloadingUrls)
@@ -823,6 +824,24 @@ void Storage::OnServerListDownloaded(vector<string> const & urls)
                                 bind(&Storage::OnMapFileDownloadProgress, this, _1));
 }
 
+void Storage::SetDeferDownloading()
+{
+  ASSERT_THREAD_CHECKER(m_threadChecker, ());
+
+  m_needToStartDeferredDownloading = true;
+}
+
+void Storage::DoDeferredDownloadIfNeeded()
+{
+  ASSERT_THREAD_CHECKER(m_threadChecker, ());
+
+  if (!m_needToStartDeferredDownloading || !m_sessionServerList)
+    return;
+
+  m_needToStartDeferredDownloading = false;
+  DoDownload();
+}
+
 void Storage::OnMapFileDownloadProgress(MapFilesDownloader::TProgress const & progress)
 {
   ASSERT_THREAD_CHECKER(m_threadChecker, ());
@@ -1083,6 +1102,7 @@ string Storage::GetLocale() const { return m_countryNameGetter.GetLocale(); }
 void Storage::SetDownloaderForTesting(unique_ptr<MapFilesDownloader> && downloader)
 {
   m_downloader = move(downloader);
+  LoadServerListForTesting();
 }
 
 void Storage::SetCurrentDataVersionForTesting(int64_t currentVersion)
@@ -1484,6 +1504,41 @@ void Storage::ApplyDiff(TCountryId const & countryId, function<void(bool isSucce
   });
 }
 
+void Storage::LoadServerListForSession()
+{
+  ASSERT_THREAD_CHECKER(m_threadChecker, ());
+
+  m_downloader->GetServersList([this](auto const & urls) { PingServerList(urls); });
+}
+
+void Storage::LoadServerListForTesting()
+{
+  ASSERT_THREAD_CHECKER(m_threadChecker, ());
+
+  m_downloader->GetServersList([this](auto const & urls) { m_sessionServerList = urls; });
+}
+
+void Storage::PingServerList(vector<string> const & urls)
+{
+  ASSERT_THREAD_CHECKER(m_threadChecker, ());
+
+  if (urls.empty())
+    return;
+
+  GetPlatform().RunTask(Platform::Thread::Network, [urls, this] {
+    Pinger::Ping(urls, [this, urls](auto readyUrls) {
+      ASSERT_THREAD_CHECKER(m_threadChecker, ());
+
+      if (readyUrls.empty())
+        m_sessionServerList = urls;
+      else
+        m_sessionServerList = move(readyUrls);
+
+      DoDeferredDownloadIfNeeded();
+    });
+  });
+}
+
 bool Storage::IsPossibleToAutoupdate() const
 {
   ASSERT_THREAD_CHECKER(m_threadChecker, ());
@@ -1514,10 +1569,7 @@ void Storage::OnDiffStatusReceived(diffs::Status const status)
     m_notAppliedDiffs.clear();
   }
 
-  for (auto const & urls : m_deferredDownloads)
-    OnServerListDownloaded(urls);
-
-  m_deferredDownloads.clear();
+  DoDeferredDownloadIfNeeded();
 }
 
 StatusAndError Storage::GetNodeStatusInfo(
diff --git a/storage/storage.hpp b/storage/storage.hpp
index b938eca52c..eb68c5c75c 100644
--- a/storage/storage.hpp
+++ b/storage/storage.hpp
@@ -7,9 +7,9 @@
 #include "storage/downloading_policy.hpp"
 #include "storage/index.hpp"
 #include "storage/map_files_downloader.hpp"
+#include "storage/pinger.hpp"
 #include "storage/queued_country.hpp"
 #include "storage/storage_defines.hpp"
-#include "storage/storage_defines.hpp"
 
 #include "platform/local_country_file.hpp"
 
@@ -17,7 +17,6 @@
 #include "base/thread_checker.hpp"
 #include "base/worker_thread.hpp"
 
-#include <future>
 #include "std/function.hpp"
 #include "std/list.hpp"
 #include "std/shared_ptr.hpp"
@@ -26,6 +25,8 @@
 #include "std/utility.hpp"
 #include "std/vector.hpp"
 
+#include <boost/optional.hpp>
+
 namespace storage
 {
 struct CountryIdAndName
@@ -250,7 +251,8 @@ private:
   diffs::Manager m_diffManager;
   vector<platform::LocalCountryFile> m_notAppliedDiffs;
 
-  vector<vector<string>> m_deferredDownloads;
+  bool m_needToStartDeferredDownloading = false;
+  boost::optional<vector<string>> m_sessionServerList;
 
   StartDownloadingCallback m_startDownloadingCallback;
 
@@ -263,9 +265,9 @@ private:
   void ReportProgressForHierarchy(TCountryId const & countryId,
                                   MapFilesDownloader::TProgress const & leafProgress);
 
-  /// Called on the main thread by MapFilesDownloader when list of
-  /// suitable servers is received.
-  void OnServerListDownloaded(vector<string> const & urls);
+  void DoDownload();
+  void SetDeferDownloading();
+  void DoDeferredDownloadIfNeeded();
 
   /// Called on the main thread by MapFilesDownloader when
   /// downloading of a map file succeeds/fails.
@@ -673,6 +675,10 @@ private:
   // Should be called once on startup, downloading process should be suspended until this method
   // was not called. Do not call this method manually.
   void OnDiffStatusReceived(diffs::Status const status) override;
+
+  void LoadServerListForSession();
+  void LoadServerListForTesting();
+  void PingServerList(vector<string> const & urls);
 };
 
 void GetQueuedCountries(Storage::TQueue const & queue, TCountriesSet & resultCountries);
diff --git a/storage/storage_tests/fake_map_files_downloader.cpp b/storage/storage_tests/fake_map_files_downloader.cpp
index ce9b3e0f68..7d6b25234b 100644
--- a/storage/storage_tests/fake_map_files_downloader.cpp
+++ b/storage/storage_tests/fake_map_files_downloader.cpp
@@ -20,13 +20,12 @@ FakeMapFilesDownloader::FakeMapFilesDownloader(TaskRunner & taskRunner)
 
 FakeMapFilesDownloader::~FakeMapFilesDownloader() { CHECK(m_checker.CalledOnOriginalThread(), ()); }
 
-void FakeMapFilesDownloader::GetServersList(int64_t const mapVersion, string const & mapFileName,
-                                            TServersListCallback const & callback)
+void FakeMapFilesDownloader::GetServersList(TServersListCallback const & callback)
 {
   CHECK(m_checker.CalledOnOriginalThread(), ());
   m_idle = false;
   MY_SCOPE_GUARD(resetIdle, bind(&FakeMapFilesDownloader::Reset, this));
-  m_taskRunner.PostTask(bind(callback, m_servers));
+  callback(m_servers);
 }
 
 void FakeMapFilesDownloader::DownloadMapFile(vector<string> const & urls, string const & path,
diff --git a/storage/storage_tests/fake_map_files_downloader.hpp b/storage/storage_tests/fake_map_files_downloader.hpp
index 9bdd4ae6f9..8c61fe9df4 100644
--- a/storage/storage_tests/fake_map_files_downloader.hpp
+++ b/storage/storage_tests/fake_map_files_downloader.hpp
@@ -26,7 +26,8 @@ public:
   virtual ~FakeMapFilesDownloader();
 
   // MapFilesDownloader overrides:
-  void GetServersList(int64_t const mapVersion, string const & mapFileName, TServersListCallback const & callback) override;
+  void GetServersList(TServersListCallback const & callback) override;
+
   void DownloadMapFile(vector<string> const & urls, string const & path, int64_t size,
                        TFileDownloadedCallback const & onDownloaded,
                        TDownloadingProgressCallback const & onProgress) override;
diff --git a/storage/storage_tests/storage_tests.cpp b/storage/storage_tests/storage_tests.cpp
index a3ae78afba..fba4184f13 100644
--- a/storage/storage_tests/storage_tests.cpp
+++ b/storage/storage_tests/storage_tests.cpp
@@ -964,7 +964,6 @@ UNIT_CLASS_TEST(TwoComponentStorageTest, DeleteCountry)
   map.Reset();
 }
 
-
 UNIT_TEST(StorageTest_FailedDownloading)
 {
   Storage storage;
diff --git a/storage/storage_tests/test_map_files_downloader.cpp b/storage/storage_tests/test_map_files_downloader.cpp
index b6fffe4a7f..e493eb92d6 100644
--- a/storage/storage_tests/test_map_files_downloader.cpp
+++ b/storage/storage_tests/test_map_files_downloader.cpp
@@ -5,8 +5,7 @@
 
 namespace storage
 {
-void TestMapFilesDownloader::GetServersList(int64_t const mapVersion, string const & mapFileName,
-                                            TServersListCallback const & callback)
+void TestMapFilesDownloader::GetServersList(TServersListCallback const & callback)
 {
   vector<string> urls = {"http://localhost:34568/unit_tests/"};
   callback(urls);
diff --git a/storage/storage_tests/test_map_files_downloader.hpp b/storage/storage_tests/test_map_files_downloader.hpp
index ada5deb7fc..1ccafc3efb 100644
--- a/storage/storage_tests/test_map_files_downloader.hpp
+++ b/storage/storage_tests/test_map_files_downloader.hpp
@@ -8,7 +8,6 @@ class TestMapFilesDownloader : public HttpMapFilesDownloader
 {
 public:
   // MapFilesDownloader overrides:
-  void GetServersList(int64_t const mapVersion, string const & mapFileName,
-                      TServersListCallback const & callback) override;
+  void GetServersList(TServersListCallback const & callback) override;
 };
 }  // namespace storage