diff options
| -rw-r--r-- | generator/coastlines_generator.cpp | 4 | ||||
| -rw-r--r-- | generator/coastlines_generator.hpp | 4 | ||||
| -rw-r--r-- | generator/feature_generator.cpp | 39 | ||||
| -rw-r--r-- | generator/generator.pro | 2 | ||||
| -rw-r--r-- | generator/generator_tests/concurrent_tests.cpp | 45 | ||||
| -rw-r--r-- | generator/generator_tests/generator_tests.pro | 1 | ||||
| -rw-r--r-- | generator/multiproducer_oneconsumer.cpp | 63 | ||||
| -rw-r--r-- | generator/multiproducer_oneconsumer.hpp | 56 | ||||
| -rw-r--r-- | generator/polygonizer.hpp | 116 |
9 files changed, 96 insertions, 234 deletions
diff --git a/generator/coastlines_generator.cpp b/generator/coastlines_generator.cpp index 03fae05902..28f3ea03d1 100644 --- a/generator/coastlines_generator.cpp +++ b/generator/coastlines_generator.cpp @@ -176,7 +176,7 @@ namespace }; } -bool CoastlineFeaturesGenerator::GetFeature(CellIdT const & cell, FeatureBuilder1 & fb) const +bool CoastlineFeaturesGenerator::GetFeature(CellIdT const & cell, FeatureBuilder1 & fb) { // get rect cell double minX, minY, maxX, maxY; @@ -211,7 +211,7 @@ bool CoastlineFeaturesGenerator::GetFeature(CellIdT const & cell, FeatureBuilder return true; } -void CoastlineFeaturesGenerator::GetFeatures(size_t i, vector<FeatureBuilder1> & vecFb) const +void CoastlineFeaturesGenerator::GetFeatures(size_t i, vector<FeatureBuilder1> & vecFb) { vector<CellIdT> stCells; stCells.push_back(CellIdT::FromBitsAndLevel(i, m_lowLevel)); diff --git a/generator/coastlines_generator.hpp b/generator/coastlines_generator.hpp index c8a4a1daa2..f52ccd52ca 100644 --- a/generator/coastlines_generator.hpp +++ b/generator/coastlines_generator.hpp @@ -22,7 +22,7 @@ class CoastlineFeaturesGenerator uint32_t m_coastType; int m_lowLevel, m_highLevel, m_maxPoints; - bool GetFeature(CellIdT const & cell, FeatureBuilder1 & fb) const; + bool GetFeature(CellIdT const & cell, FeatureBuilder1 & fb); public: CoastlineFeaturesGenerator(uint32_t coastType, @@ -34,5 +34,5 @@ public: void Finish(); inline size_t GetCellsCount() const { return 1 << 2 * m_lowLevel; } - void GetFeatures(size_t i, vector<FeatureBuilder1> & vecFb) const; + void GetFeatures(size_t i, vector<FeatureBuilder1> & vecFb); }; diff --git a/generator/feature_generator.cpp b/generator/feature_generator.cpp index a5592fb343..4e23c671ac 100644 --- a/generator/feature_generator.cpp +++ b/generator/feature_generator.cpp @@ -6,7 +6,6 @@ #include "generate_info.hpp" #include "coastlines_generator.hpp" #include "world_map_generator.hpp" -#include "multiproducer_oneconsumer.hpp" #include "../defines.hpp" @@ -336,32 +335,6 @@ public: } } -private: - class CoastFeatureTask : public MultiProducerOneConsumer::ITask - { - MainFeaturesEmitter & m_parent; - size_t m_ind; - - public: - CoastFeatureTask(MainFeaturesEmitter & parent, size_t ind) - : m_parent(parent), m_ind(ind) {} - - virtual void RunBase() - { - vector<FeatureBuilder1> vecFb; - m_parent.m_coasts->GetFeatures(m_ind, vecFb); - - for (size_t i = 0; i< vecFb.size(); ++i) - Emit(&vecFb[i]); - } - - virtual void EmitBase(void * p) - { - (*m_parent.m_coastsHolder)(*reinterpret_cast<FeatureBuilder1 *>(p)); - } - }; - -public: void Finish() { if (m_world) @@ -372,12 +345,16 @@ public: m_coasts->Finish(); size_t const count = m_coasts->GetCellsCount(); - LOG(LINFO, ("Generating coastline features for ", count, " cells.")); + LOG(LINFO, ("Generating coastline polygons", count)); - MultiProducerOneConsumer runner(8); for (size_t i = 0; i < count; ++i) - runner.RunTask(new CoastFeatureTask(*this, i)); - runner.Finish(); + { + vector<FeatureBuilder1> vecFb; + m_coasts->GetFeatures(i, vecFb); + + for (size_t j = 0; j < vecFb.size(); ++j) + (*m_coastsHolder)(vecFb[j]); + } } else if (m_coastsHolder) { diff --git a/generator/generator.pro b/generator/generator.pro index 86ab0203d9..aa65d5dd8b 100644 --- a/generator/generator.pro +++ b/generator/generator.pro @@ -30,7 +30,6 @@ SOURCES += \ osm_decl.cpp \ coastlines_generator.cpp \ tesselator.cpp \ - multiproducer_oneconsumer.cpp \ HEADERS += \ feature_merger.hpp \ @@ -58,4 +57,3 @@ HEADERS += \ osm_decl.hpp \ coastlines_generator.hpp \ tesselator.hpp \ - multiproducer_oneconsumer.hpp \ diff --git a/generator/generator_tests/concurrent_tests.cpp b/generator/generator_tests/concurrent_tests.cpp deleted file mode 100644 index fdc90ecaa9..0000000000 --- a/generator/generator_tests/concurrent_tests.cpp +++ /dev/null @@ -1,45 +0,0 @@ -#include "../../testing/testing.hpp" - -#include "../multiproducer_oneconsumer.hpp" - - -namespace -{ - class AccumulateTask : public MultiProducerOneConsumer::ITask - { - vector<uint64_t> & m_res; - public: - AccumulateTask(vector<uint64_t> & res) : m_res(res) {} - - static uint64_t const s_upper = 1000000; - - virtual void RunBase() - { - uint64_t summ = 0; - for (uint64_t i = 0; i <= s_upper; ++i) - summ += i; - Emit(reinterpret_cast<void *>(summ)); - } - - virtual void EmitBase(void * p) - { - m_res.push_back(reinterpret_cast<uint64_t>(p)); - } - }; -} - -UNIT_TEST(MultiProducers_Smoke) -{ - size_t const count = 64; - vector<uint64_t> vec; - - MultiProducerOneConsumer runner(8); - for (size_t i = 0; i < count; ++i) - runner.RunTask(new AccumulateTask(vec)); - runner.Finish(); - - TEST_EQUAL(vec.size(), count, ()); - uint64_t const res = AccumulateTask::s_upper * (AccumulateTask::s_upper + 1) / 2; - for (size_t i = 0; i < count; ++i) - TEST_EQUAL(vec[i], res, ()); -} diff --git a/generator/generator_tests/generator_tests.pro b/generator/generator_tests/generator_tests.pro index a715efeaa8..c125655967 100644 --- a/generator/generator_tests/generator_tests.pro +++ b/generator/generator_tests/generator_tests.pro @@ -27,5 +27,4 @@ SOURCES += \ tesselator_test.cpp \ triangles_tree_coding_test.cpp \ coasts_test.cpp \ - concurrent_tests.cpp \ feature_builder_test.cpp \ diff --git a/generator/multiproducer_oneconsumer.cpp b/generator/multiproducer_oneconsumer.cpp deleted file mode 100644 index 1146be6498..0000000000 --- a/generator/multiproducer_oneconsumer.cpp +++ /dev/null @@ -1,63 +0,0 @@ -#include "multiproducer_oneconsumer.hpp" - -#include "../base/logging.hpp" -#include "../base/macros.hpp" -#include "../base/assert.hpp" - - -MultiProducerOneConsumer::MultiProducerOneConsumer(size_t tasksPerThread) -#if PARALLEL_POLYGONIZER - : m_ThreadPoolSemaphore(m_ThreadPool.maxThreadCount() * tasksPerThread) -#endif -{ -#if PARALLEL_POLYGONIZER - LOG(LINFO, ("QThreadPool threads count = ", m_ThreadPool.maxThreadCount())); -#endif -} - -void MultiProducerOneConsumer::RunTask(ITask * pTask) -{ -#if PARALLEL_POLYGONIZER - pTask->BeforeStart(this); - m_ThreadPool.start(pTask); - -#else - pTask->RunBase(); - delete pTask; - -#endif -} - -#if PARALLEL_POLYGONIZER -void MultiProducerOneConsumer::ITask::BeforeStart(MultiProducerOneConsumer * pParent) -{ - ASSERT ( pParent, () ); - m_pParent = pParent; - m_pParent->m_ThreadPoolSemaphore.acquire(); -} - -void MultiProducerOneConsumer::ITask::run() -{ - RunBase(); - - m_pParent->m_ThreadPoolSemaphore.release(); -} -#endif - -void MultiProducerOneConsumer::ITask::Emit(void * p) -{ -#if PARALLEL_POLYGONIZER - ASSERT ( m_pParent, () ); - QMutexLocker mutexLocker(&m_pParent->m_EmitMutex); - UNUSED_VALUE(mutexLocker); -#endif - - EmitBase(p); -} - -void MultiProducerOneConsumer::Finish() -{ -#if PARALLEL_POLYGONIZER - m_ThreadPool.waitForDone(); -#endif -} diff --git a/generator/multiproducer_oneconsumer.hpp b/generator/multiproducer_oneconsumer.hpp deleted file mode 100644 index 0df2decf76..0000000000 --- a/generator/multiproducer_oneconsumer.hpp +++ /dev/null @@ -1,56 +0,0 @@ -#pragma once - -#ifndef PARALLEL_POLYGONIZER -#define PARALLEL_POLYGONIZER 1 -#endif - -#if PARALLEL_POLYGONIZER -#include <QSemaphore> -#include <QThreadPool> -#include <QMutex> -#include <QMutexLocker> -#endif - - -class MultiProducerOneConsumer -{ -#if PARALLEL_POLYGONIZER - QThreadPool m_ThreadPool; - QSemaphore m_ThreadPoolSemaphore; - QMutex m_EmitMutex; -#endif - -public: - MultiProducerOneConsumer(size_t tasksPerThread); - - class ITask -#if PARALLEL_POLYGONIZER - : public QRunnable -#endif - { -#if PARALLEL_POLYGONIZER - MultiProducerOneConsumer * m_pParent; -#endif - - public: - ITask() : m_pParent(0) {} - -#if PARALLEL_POLYGONIZER - void BeforeStart(MultiProducerOneConsumer * pParent); -#endif - - virtual void RunBase() = 0; - virtual void EmitBase(void * p) = 0; - - void Emit(void * p); - -#if PARALLEL_POLYGONIZER - // Override - virtual void run(); -#endif - }; - - void RunTask(ITask * pTask); - - void Finish(); -}; diff --git a/generator/polygonizer.hpp b/generator/polygonizer.hpp index 35d941c199..3602224562 100644 --- a/generator/polygonizer.hpp +++ b/generator/polygonizer.hpp @@ -2,19 +2,39 @@ #include "borders_loader.hpp" #include "feature_builder.hpp" -#include "multiproducer_oneconsumer.hpp" + +#include "../indexer/feature_visibility.hpp" +#include "../indexer/cell_id.hpp" #include "../geometry/rect2d.hpp" +#include "../coding/file_writer.hpp" + +#include "../base/base.hpp" #include "../base/buffer_vector.hpp" +#include "../base/macros.hpp" +#include "../std/scoped_ptr.hpp" #include "../std/string.hpp" +#ifndef PARALLEL_POLYGONIZER +#define PARALLEL_POLYGONIZER 1 +#endif + +#if PARALLEL_POLYGONIZER +#include <QSemaphore> +#include <QThreadPool> +#include <QMutex> +#include <QMutexLocker> +#endif + + namespace feature { // Groups features according to country polygons - template <class FeatureOutT> class Polygonizer + template <class FeatureOutT> + class Polygonizer { string m_prefix; string m_suffix; @@ -23,13 +43,24 @@ namespace feature vector<string> m_Names; borders::CountriesContainerT m_countries; - MultiProducerOneConsumer m_impl; +#if PARALLEL_POLYGONIZER + QThreadPool m_ThreadPool; + QSemaphore m_ThreadPoolSemaphore; + QMutex m_EmitFeatureMutex; +#endif public: template <class TInfo> explicit Polygonizer(TInfo const & info) - : m_prefix(info.m_datFilePrefix), m_suffix(info.m_datFileSuffix), m_impl(8) + : m_prefix(info.m_datFilePrefix), m_suffix(info.m_datFileSuffix) +#if PARALLEL_POLYGONIZER + , m_ThreadPoolSemaphore(m_ThreadPool.maxThreadCount() * 8) +#endif { +#if PARALLEL_POLYGONIZER + LOG(LINFO, ("Polygonizer thread pool threads:", m_ThreadPool.maxThreadCount())); +#endif + if (info.m_splitByPolygons) { CHECK(borders::LoadCountriesList(info.m_datFilePrefix, m_countries), @@ -44,7 +75,7 @@ namespace feature } ~Polygonizer() { - m_impl.Finish(); + Finish(); for_each(m_Buckets.begin(), m_Buckets.end(), DeleteFunctor()); } @@ -69,16 +100,14 @@ namespace feature } }; - typedef borders::CountryPolygons PolygonsT; - typedef buffer_vector<PolygonsT const *, 32> PolygonsVectorT; - class InsertCountriesPtr { - PolygonsVectorT & m_vec; + typedef buffer_vector<borders::CountryPolygons const *, 32> vec_type; + vec_type & m_vec; public: - InsertCountriesPtr(PolygonsVectorT & vec) : m_vec(vec) {} - void operator() (PolygonsT const & c) + InsertCountriesPtr(vec_type & vec) : m_vec(vec) {} + void operator() (borders::CountryPolygons const & c) { m_vec.push_back(&c); } @@ -86,7 +115,7 @@ namespace feature void operator () (FeatureBuilder1 const & fb) { - PolygonsVectorT vec; + buffer_vector<borders::CountryPolygons const *, 32> vec; m_countries.ForEachInRect(fb.GetLimitRect(), InsertCountriesPtr(vec)); switch (vec.size()) @@ -97,13 +126,31 @@ namespace feature EmitFeature(vec[0], fb); break; default: - m_impl.RunTask(new PolygonizerTask(*this, vec, fb)); - break; + { +#if PARALLEL_POLYGONIZER + m_ThreadPoolSemaphore.acquire(); + m_ThreadPool.start(new PolygonizerTask(this, vec, fb)); +#else + PolygonizerTask task(this, vec, fb); + task.RunBase(); +#endif + } } } - void EmitFeature(PolygonsT const * country, FeatureBuilder1 const & fb) + void Finish() + { +#if PARALLEL_POLYGONIZER + m_ThreadPool.waitForDone(); +#endif + } + + void EmitFeature(borders::CountryPolygons const * country, FeatureBuilder1 const & fb) { +#if PARALLEL_POLYGONIZER + QMutexLocker mutexLocker(&m_EmitFeatureMutex); + UNUSED_VALUE(mutexLocker); +#endif if (country->m_index == -1) { m_Names.push_back(country->m_name); @@ -114,20 +161,26 @@ namespace feature (*(m_Buckets[country->m_index]))(fb); } - inline vector<string> const & Names() const { return m_Names; } + vector<string> const & Names() const + { + return m_Names; + } private: + friend class PolygonizerTask; - class PolygonizerTask : public MultiProducerOneConsumer::ITask + class PolygonizerTask +#if PARALLEL_POLYGONIZER + : public QRunnable +#endif { public: - PolygonizerTask(Polygonizer & polygonizer, - PolygonsVectorT const & countries, + PolygonizerTask(Polygonizer * pPolygonizer, + buffer_vector<borders::CountryPolygons const *, 32> const & countries, FeatureBuilder1 const & fb) - : m_polygonizer(polygonizer), m_Countries(countries), m_FB(fb) {} + : m_pPolygonizer(pPolygonizer), m_Countries(countries), m_FB(fb) {} - // Override - virtual void RunBase() + void RunBase() { for (size_t i = 0; i < m_Countries.size(); ++i) { @@ -135,24 +188,23 @@ namespace feature m_FB.ForEachGeometryPoint(doCheck); if (doCheck.m_belongs) - Emit(const_cast<PolygonsT *>(m_Countries[i])); + m_pPolygonizer->EmitFeature(m_Countries[i], m_FB); } } - // Override - virtual void EmitBase(void * p) +#if PARALLEL_POLYGONIZER + void run() { - m_polygonizer.EmitFeature(reinterpret_cast<PolygonsT const *>(p), m_FB); + RunBase(); + + m_pPolygonizer->m_ThreadPoolSemaphore.release(); } +#endif private: - Polygonizer & m_polygonizer; - - /// @name Do copy of all input parameters. - //@{ - PolygonsVectorT m_Countries; + Polygonizer * m_pPolygonizer; + buffer_vector<borders::CountryPolygons const *, 32> m_Countries; FeatureBuilder1 m_FB; - //@} }; }; } |