Added generator_tool option for set osm data type [xml or o5m]

13 files changed, 749 insertions, 708 deletions

diff --git a/generator/data_generator.cpp b/generator/data_generator.cpp
deleted file mode 100644
index 39702af888..0000000000
--- a/generator/data_generator.cpp
+++ /dev/null
@@ -1,107 +0,0 @@
-#include "data_generator.hpp"
-#include "data_cache_file.hpp"
-#include "first_pass_parser.hpp"
-#include "osm_decl.hpp"
-
-#include "../base/std_serialization.hpp"
-#include "../base/logging.hpp"
-
-#include "../std/bind.hpp"
-#include "point_storage.hpp"
-
-namespace data
-{
-
-template <class TNodesHolder>
-class FileHolder : public cache::BaseFileHolder<TNodesHolder, cache::DataFileWriter, FileWriter>
-{
-  typedef cache::BaseFileHolder<TNodesHolder, cache::DataFileWriter, FileWriter> base_type;
-
-  typedef typename base_type::user_id_t user_id_t;
-
-  template <class TMap, class TVec>
-  void add_id2rel_vector(TMap & rMap, user_id_t relid, TVec const & v)
-  {
-    for (size_t i = 0; i < v.size(); ++i)
-      rMap.write(v[i].first, relid);
-  }
-
-public:
-  FileHolder(TNodesHolder & nodes, string const & dir) : base_type(nodes, dir) {}
-
-  void AddNode(uint64_t id, double lat, double lng)
-  {
-    this->m_nodes.AddPoint(id, lat, lng);
-  }
-
-  void AddWay(user_id_t id, WayElement const & e)
-  {
-    this->m_ways.Write(id, e);
-  }
-
-  void AddRelation(user_id_t id, RelationElement const & e)
-  {
-    const string relationType = e.GetType();
-    if (relationType == "multipolygon" || relationType == "route" || relationType == "boundary")
-    {
-      this->m_relations.Write(id, e);
-
-      add_id2rel_vector(this->m_nodes2rel, id, e.nodes);
-      add_id2rel_vector(this->m_ways2rel, id, e.ways);
-    }
-  }
-
-  void SaveIndex()
-  {
-    this->m_ways.SaveOffsets();
-    this->m_relations.SaveOffsets();
-
-    this->m_nodes2rel.flush_to_file();
-    this->m_ways2rel.flush_to_file();
-  }
-};
-
-template <class TNodesHolder>
-bool GenerateImpl(string const & dir, string const & osmFileName = string())
-{
-  try
-  {
-    TNodesHolder nodes(dir + NODES_FILE);
-    typedef FileHolder<TNodesHolder> holder_t;
-    holder_t holder(nodes, dir);
-
-    FirstPassParser<holder_t> parser(holder);
-    if (osmFileName.empty())
-      ParseXMLFromStdIn(parser);
-    else
-      ParseXMLFromFile(parser, osmFileName);
-
-    LOG(LINFO, ("Added points count = ", nodes.GetCount()));
-
-    holder.SaveIndex();
-  }
-  catch (Writer::Exception const & e)
-  {
-    LOG(LERROR, ("Error with file ", e.what()));
-    return false;
-  }
-
-  return true;
-}
-
-bool GenerateToFile(string const & dir, string const & nodeStorage, string const & osmFileName)
-{
-  if (nodeStorage == "raw")
-    return GenerateImpl<RawFileShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileName);
-  else if (nodeStorage == "map")
-    return GenerateImpl<MapFileShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileName);
-  else if (nodeStorage == "sqlite")
-    return GenerateImpl<SQLitePointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileName);
-  else if (nodeStorage == "mem")
-    return GenerateImpl<RawMemShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileName);
-  else
-    CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage));
-  return false;
-}
-
-}
diff --git a/generator/data_generator.hpp b/generator/data_generator.hpp
deleted file mode 100644
index 424308a8a6..0000000000
--- a/generator/data_generator.hpp
+++ /dev/null
@@ -1,8 +0,0 @@
-#pragma once
-
-#include "../std/string.hpp"
-
-namespace data
-{
-  bool GenerateToFile(string const & dir, string const & nodeStorage, string const & osmFileName);
-}
diff --git a/generator/feature_generator.cpp b/generator/feature_generator.cpp
index c4c616d129..f74d68192b 100644
--- a/generator/feature_generator.cpp
+++ b/generator/feature_generator.cpp
@@ -1,18 +1,13 @@
 #include "feature_generator.hpp"
 #include "data_cache_file.hpp"
 #include "osm_element.hpp"
-#include "polygonizer.hpp"
+
 #include "osm_decl.hpp"
 #include "generate_info.hpp"
-#include "coastlines_generator.hpp"
-#include "world_map_generator.hpp"
-
-#include "../defines.hpp"
 
 #include "../indexer/data_header.hpp"
 #include "../indexer/mercator.hpp"
 #include "../indexer/cell_id.hpp"
-#include "../indexer/classificator.hpp"
 
 #include "../coding/varint.hpp"
 
@@ -24,103 +19,11 @@
 #include "../std/unordered_map.hpp"
 #include "../std/target_os.hpp"
 
-#include "point_storage.hpp"
-
-namespace feature
-{
-
-template <class TNodesHolder>
-class FileHolder : public cache::BaseFileHolder<TNodesHolder, cache::DataFileReader, FileReader>
-{
-  typedef cache::DataFileReader reader_t;
-  typedef cache::BaseFileHolder<TNodesHolder, reader_t, FileReader> base_type;
-
-  typedef typename base_type::offset_map_t offset_map_t;
-
-  typedef typename base_type::user_id_t user_id_t;
-
-  template <class TElement, class ToDo> struct process_base
-  {
-  protected:
-    reader_t & m_reader;
-    ToDo & m_toDo;
-  public:
-    process_base(reader_t & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {}
-
-    bool operator() (uint64_t id)
-    {
-      TElement e;
-      if (m_reader.Read(id, e))
-        return m_toDo(id, e);
-      return false;
-    }
-  };
-
-  template <class ToDo> struct process_relation : public process_base<RelationElement, ToDo>
-  {
-    typedef process_base<RelationElement, ToDo> base_type;
-  public:
-    process_relation(reader_t & reader, ToDo & toDo) : base_type(reader, toDo) {}
-  };
-
-  template <class ToDo> struct process_relation_cached : public process_relation<ToDo>
-  {
-    typedef process_relation<ToDo> base_type;
-
-  public:
-    process_relation_cached(reader_t & rels, ToDo & toDo)
-      : base_type(rels, toDo) {}
-
-    bool operator() (uint64_t id)
-    {
-      return this->m_toDo(id, this->m_reader);
-    }
-  };
-
-public:
-  FileHolder(TNodesHolder & holder, string const & dir) : base_type(holder, dir) {}
-
-  bool GetNode(uint64_t id, double & lat, double & lng)
-  {
-    return this->m_nodes.GetPoint(id, lat, lng);
-  }
-
-  bool GetWay(user_id_t id, WayElement & e)
-  {
-    return this->m_ways.Read(id, e);
-  }
-
-  template <class ToDo> void ForEachRelationByWay(user_id_t id, ToDo & toDo)
-  {
-    process_relation<ToDo> processor(this->m_relations, toDo);
-    this->m_ways2rel.for_each_ret(id, processor);
-  }
-
-  template <class ToDo> void ForEachRelationByNodeCached(user_id_t id, ToDo & toDo)
-  {
-    process_relation_cached<ToDo> processor(this->m_relations, toDo);
-    this->m_nodes2rel.for_each_ret(id, processor);
-  }
-
-  template <class ToDo> void ForEachRelationByWayCached(user_id_t id, ToDo & toDo)
-  {
-    process_relation_cached<ToDo> processor(this->m_relations, toDo);
-    this->m_ways2rel.for_each_ret(id, processor);
-  }
-
-  void LoadIndex()
-  {
-    this->m_ways.LoadOffsets();
-    this->m_relations.LoadOffsets();
-
-    this->m_nodes2rel.read_to_memory();
-    this->m_ways2rel.read_to_memory();
-  }
-};
 
 ///////////////////////////////////////////////////////////////////////////////////////////////////
 // FeaturesCollector implementation
 ///////////////////////////////////////////////////////////////////////////////////////////////////
+namespace feature {
 
 FeaturesCollector::FeaturesCollector(string const & fName)
 : m_datFile(fName)
@@ -213,232 +116,5 @@ void FeaturesCollector::operator() (FeatureBuilder1 const & fb)
   (void)WriteFeatureBase(bytes, fb);
 }
 
-///////////////////////////////////////////////////////////////////////////////////////////////////
-// Generate functions implementations.
-///////////////////////////////////////////////////////////////////////////////////////////////////
-
-namespace
-{
-
-class MainFeaturesEmitter
-{
-  typedef WorldMapGenerator<FeaturesCollector> WorldGenerator;
-  typedef CountryMapGenerator<Polygonizer<FeaturesCollector> > CountriesGenerator;
-  unique_ptr<CountriesGenerator> m_countries;
-  unique_ptr<WorldGenerator> m_world;
-
-  unique_ptr<CoastlineFeaturesGenerator> m_coasts;
-  unique_ptr<FeaturesCollector> m_coastsHolder;
-
-  string m_srcCoastsFile;
-
-  template <class T1, class T2> class CombinedEmitter
-  {
-    T1 * m_p1;
-    T2 * m_p2;
-  public:
-    CombinedEmitter(T1 * p1, T2 * p2) : m_p1(p1), m_p2(p2) {}
-    void operator() (FeatureBuilder1 const & fb, uint64_t)
-    {
-      if (m_p1) (*m_p1)(fb);
-      if (m_p2) (*m_p2)(fb);
-    }
-  };
-
-  enum TypeIndex
-  {
-    NATURAL_COASTLINE,
-    NATURAL_LAND,
-    PLACE_ISLAND,
-    PLACE_ISLET,
-
-    TYPES_COUNT
-  };
-  uint32_t m_types[TYPES_COUNT];
-
-  inline uint32_t Type(TypeIndex i) const { return m_types[i]; }
-
-public:
-  MainFeaturesEmitter(GenerateInfo const & info)
-  {
-    Classificator const & c = classif();
-
-    char const * arr[][2] = {
-      { "natural", "coastline" },
-      { "natural", "land" },
-      { "place", "island" },
-      { "place", "islet" }
-    };
-    STATIC_ASSERT(ARRAY_SIZE(arr) == TYPES_COUNT);
-
-    for (size_t i = 0; i < ARRAY_SIZE(arr); ++i)
-      m_types[i] = c.GetTypeByPath(vector<string>(arr[i], arr[i] + 2));
-
-    m_srcCoastsFile = info.m_tmpDir + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix;
-
-    CHECK(!info.m_makeCoasts || !info.m_createWorld,
-          ("We can't do make_coasts and generate_world at the same time"));
-
-    if (!info.m_makeCoasts)
-    {
-      m_countries.reset(new CountriesGenerator(info));
-
-      if (info.m_emitCoasts)
-      {
-        m_coastsHolder.reset(new FeaturesCollector(
-                info.m_datFilePrefix + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix));
-      }
-    }
-    else
-    {
-      // 4-10 - level range for cells
-      // 20000 - max points count per feature
-      m_coasts.reset(new CoastlineFeaturesGenerator(Type(NATURAL_COASTLINE), 4, 10, 20000));
-
-      m_coastsHolder.reset(new FeaturesCollector(m_srcCoastsFile));
-    }
-
-    if (info.m_createWorld)
-      m_world.reset(new WorldGenerator(info));
-  }
-
-  void operator() (FeatureBuilder1 fb)
-  {
-    uint32_t const coastType = Type(NATURAL_COASTLINE);
-    bool const hasCoast = fb.HasType(coastType);
-
-    if (m_coasts)
-    {
-      if (hasCoast)
-      {
-        CHECK ( fb.GetGeomType() != feature::GEOM_POINT, () );
-
-        // leave only coastline type
-        fb.SetType(coastType);
-        (*m_coasts)(fb);
-      }
-    }
-    else
-    {
-      if (hasCoast)
-      {
-        fb.PopExactType(Type(NATURAL_LAND));
-        fb.PopExactType(coastType);
-      }
-      else if ((fb.HasType(Type(PLACE_ISLAND)) || fb.HasType(Type(PLACE_ISLET))) &&
-               fb.GetGeomType() == feature::GEOM_AREA)
-      {
-        fb.AddType(Type(NATURAL_LAND));
-      }
-
-      if (fb.RemoveInvalidTypes())
-      {
-        if (m_world)
-          (*m_world)(fb);
-
-        if (m_countries)
-          (*m_countries)(fb);
-      }
-    }
-  }
-
-  /// @return false if coasts are not merged and FLAG_fail_on_coasts is set
-  bool Finish()
-  {
-    if (m_world)
-      m_world->DoMerge();
-
-    if (m_coasts)
-    {
-      // Check and stop if some coasts were not merged
-      if (!m_coasts->Finish())
-        return false;
-
-      size_t const count = m_coasts->GetCellsCount();
-      LOG(LINFO, ("Generating coastline polygons", count));
-
-      for (size_t i = 0; i < count; ++i)
-      {
-        vector<FeatureBuilder1> vecFb;
-        m_coasts->GetFeatures(i, vecFb);
-
-        for (size_t j = 0; j < vecFb.size(); ++j)
-          (*m_coastsHolder)(vecFb[j]);
-      }
-    }
-    else if (m_coastsHolder)
-    {
-      CombinedEmitter<FeaturesCollector, CountriesGenerator>
-          emitter(m_coastsHolder.get(), m_countries.get());
-      feature::ForEachFromDatRawFormat(m_srcCoastsFile, emitter);
-    }
-
-    return true;
-  }
-
-  inline void GetNames(vector<string> & names) const
-  {
-    if (m_countries)
-      names = m_countries->Parent().Names();
-    else
-      names.clear();
-  }
-};
-
-}
-
-template <class NodesHolderT>
-bool GenerateImpl(GenerateInfo & info, string const & osmFileName = string())
-{
-  try
-  {
-    NodesHolderT nodes(info.m_tmpDir + NODES_FILE);
-
-    typedef FileHolder<NodesHolderT> HolderT;
-    HolderT holder(nodes, info.m_tmpDir);
-    holder.LoadIndex();
-
-    MainFeaturesEmitter bucketer(info);
-    SecondPassParser<MainFeaturesEmitter, HolderT> parser(
-          bucketer, holder,
-          info.m_makeCoasts ? classif().GetCoastType() : 0,
-          info.m_addressFile);
-
-    if (osmFileName.empty())
-      ParseXMLFromStdIn(parser);
-    else
-      ParseXMLFromFile(parser, osmFileName);
-
-    parser.Finish();
-
-    // Stop if coasts are not merged and FLAG_fail_on_coasts is set
-    if (!bucketer.Finish())
-      return false;
-
-    bucketer.GetNames(info.m_bucketNames);
-  }
-  catch (Reader::Exception const & e)
-  {
-    LOG(LERROR, ("Error with file ", e.what()));
-    return false;
-  }
-
-  return true;
-}
-
-bool GenerateFeatures(GenerateInfo & info, string const & nodeStorage, string const & osmFileName)
-{
-  if (nodeStorage == "raw")
-    return GenerateImpl<RawFileShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileName);
-  else if (nodeStorage == "map")
-    return GenerateImpl<MapFileShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileName);
-  else if (nodeStorage == "sqlite")
-    return GenerateImpl<SQLitePointStorage<BasePointStorage::MODE_READ>>(info, osmFileName);
-  else if (nodeStorage == "mem")
-    return GenerateImpl<RawMemShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileName);
-  else
-    CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage));
-  return false;
-}
 
 }
diff --git a/generator/feature_generator.hpp b/generator/feature_generator.hpp
index c0323940f3..8ff2f66956 100644
--- a/generator/feature_generator.hpp
+++ b/generator/feature_generator.hpp
@@ -12,10 +12,6 @@ class FeatureBuilder1;
 
 namespace feature
 {
-  struct GenerateInfo;
-
-  bool GenerateFeatures(GenerateInfo & info, string const & nodeStorage, string const & osmFileName);
-
   // Writes features to dat file.
   class FeaturesCollector
   {
diff --git a/generator/generator.pro b/generator/generator.pro
index 79e1a12343..711f292870 100644
--- a/generator/generator.pro
+++ b/generator/generator.pro
@@ -16,7 +16,7 @@ QT *= core
 SOURCES += \
     feature_merger.cpp \
     xml_element.cpp \
-    data_generator.cpp \
+    osm_source.cpp \
     feature_generator.cpp \
     feature_sorter.cpp \
     update_generator.cpp \
diff --git a/generator/generator_tool/generator_tool.cpp b/generator/generator_tool/generator_tool.cpp
index 330375d818..820023f7e3 100644
--- a/generator/generator_tool/generator_tool.cpp
+++ b/generator/generator_tool/generator_tool.cpp
@@ -1,4 +1,3 @@
-#include "../data_generator.hpp"
 #include "../feature_generator.hpp"
 #include "../feature_sorter.hpp"
 #include "../update_generator.hpp"
@@ -10,6 +9,7 @@
 #include "../generate_info.hpp"
 #include "../check_model.hpp"
 #include "../routing_generator.hpp"
+#include "../osm_source.hpp"
 
 #include "../../indexer/drawing_rules.hpp"
 #include "../../indexer/classificator_loader.hpp"
@@ -35,7 +35,7 @@ DEFINE_bool(generate_update, false,
 
 DEFINE_bool(generate_classif, false, "Generate classificator.");
 
-DEFINE_bool(preprocess_xml, false, "1st pass - create nodes/ways/relations data");
+DEFINE_bool(preprocess, false, "1st pass - create nodes/ways/relations data");
 DEFINE_bool(make_coasts, false, "create intermediate file with coasts data");
 DEFINE_bool(emit_coasts, false, "push coasts features from intermediate file to out files/countries");
 
@@ -44,7 +44,7 @@ DEFINE_bool(generate_geometry, false, "3rd pass - split and simplify geometry an
 DEFINE_bool(generate_index, false, "4rd pass - generate index");
 DEFINE_bool(generate_search_index, false, "5th pass - generate search index");
 DEFINE_bool(calc_statistics, false, "Calculate feature statistics for specified mwm bucket files");
-DEFINE_string(node_storage, "raw", "Type of storage for intermediate points representation. Available: raw, map, sqlite, mem");
+DEFINE_string(node_storage, "map", "Type of storage for intermediate points representation. Available: raw, map, mem");
 DEFINE_string(data_path, "", "Working directory, 'path_to_exe/../../data' if empty.");
 DEFINE_string(output, "", "File name for process (without 'mwm' ext).");
 DEFINE_string(intermediate_data_path, "", "Path to stored nodes, ways, relations.");
@@ -65,6 +65,7 @@ DEFINE_string(address_file_name, "", "Output file name for storing full addresse
 DEFINE_string(export_poly_path, "", "Output dir for osm .poly files created from .borders (countires are read from polygons.lst)");
 DEFINE_string(osrm_file_name, "", "Input osrm file to generate routing info");
 DEFINE_string(osm_file_name, "", "Input osm area file");
+DEFINE_string(osm_file_type, "xml", "Input osm area file type [xml, o5m]");
 DEFINE_string(user_resource_path, "", "User defined resource path for classificator.txt and etc.");
 
 
@@ -101,10 +102,10 @@ int main(int argc, char ** argv)
       FLAGS_data_path.empty() ? pl.WritableDir() : AddSlashIfNeeded(FLAGS_data_path);
 
   // Generating intermediate files
-  if (FLAGS_preprocess_xml)
+  if (FLAGS_preprocess)
   {
     LOG(LINFO, ("Generating intermediate data ...."));
-    if (!data::GenerateToFile(FLAGS_intermediate_data_path, FLAGS_node_storage, FLAGS_osm_file_name))
+    if (!GenerateIntermediateData(FLAGS_intermediate_data_path, FLAGS_node_storage, FLAGS_osm_file_type, FLAGS_osm_file_name))
       return -1;
   }
 
@@ -148,7 +149,7 @@ int main(int argc, char ** argv)
     if (!FLAGS_address_file_name.empty())
       genInfo.m_addressFile = path + FLAGS_address_file_name;
 
-    if (!feature::GenerateFeatures(genInfo, FLAGS_node_storage, FLAGS_osm_file_name))
+    if (!GenerateFeatures(genInfo, FLAGS_node_storage, FLAGS_osm_file_type, FLAGS_osm_file_name))
       return -1;
 
     // without --spit_by_polygons, we have empty name country as result - assign it
diff --git a/generator/generator_tool/generator_tool.pro b/generator/generator_tool/generator_tool.pro
index 9735eea9c7..6ac03d0285 100644
--- a/generator/generator_tool/generator_tool.pro
+++ b/generator/generator_tool/generator_tool.pro
@@ -2,7 +2,7 @@
 
 ROOT_DIR = ../..
 DEPENDENCIES = generator routing storage indexer platform geometry coding base \
-               osrm gflags expat sgitess jansson protobuf tomcrypt
+               osrm gflags expat sgitess jansson protobuf tomcrypt o5mreader
 
 include($$ROOT_DIR/common.pri)
 
@@ -18,8 +18,6 @@ QT *= core
 win32: LIBS *= -lShell32
 macx-*: LIBS *= "-framework Foundation"
 
-LIBS *= -lsqlite3
-
 SOURCES += \
     generator_tool.cpp \
 
diff --git a/generator/osm_element.hpp b/generator/osm_element.hpp
index c4eea48969..4ed82bc531 100644
--- a/generator/osm_element.hpp
+++ b/generator/osm_element.hpp
@@ -382,6 +382,7 @@ class SecondPassParser : public BaseOSMParser
   }
   //@}
 
+public:
   /// The main entry point for parsing process.
   virtual void EmitElement(XMLElement * p)
   {
diff --git a/generator/osm_source.cpp b/generator/osm_source.cpp
new file mode 100644
index 0000000000..825773e995
--- /dev/null
+++ b/generator/osm_source.cpp
@@ -0,0 +1,647 @@
+#include "osm_source.hpp"
+
+#include "osm_decl.hpp"
+#include "data_cache_file.hpp"
+
+#include "coastlines_generator.hpp"
+#include "world_map_generator.hpp"
+#include "feature_generator.hpp"
+#include "polygonizer.hpp"
+
+#include "point_storage.hpp"
+
+#include "xml_element.hpp"
+
+#include "first_pass_parser.hpp"
+#include "osm_element.hpp"
+
+#include "../defines.hpp"
+#include "../indexer/mercator.hpp"
+#include "../indexer/classificator.hpp"
+
+#include "../coding/parse_xml.hpp"
+
+#include "../3party/o5mreader/o5mreader.h"
+
+
+#define DECODE_O5M_COORD(coord) (static_cast<double>(coord) / 1E+7)
+
+namespace
+{
+  class SourceReader
+  {
+    string const &m_filename;
+    FILE * m_file;
+
+  public:
+    SourceReader(string const & filename) : m_filename(filename)
+    {
+      if (m_filename.empty())
+      {
+        LOG(LINFO, ("Read OSM data from stdin..."));
+        m_file = freopen(NULL, "rb", stdin);
+      }
+      else
+      {
+        LOG(LINFO, ("Read OSM data from", filename));
+        m_file = fopen(filename.c_str(), "rb");
+      }
+    }
+
+    ~SourceReader()
+    {
+      if (!m_filename.empty())
+        fclose(m_file);
+    }
+
+    inline FILE * Handle() { return m_file; }
+
+    uint64_t Read(char * buffer, uint64_t bufferSize)
+    {
+      return fread(buffer, sizeof(char), bufferSize, m_file);
+    }
+  };
+}
+
+
+namespace feature
+{
+
+  template <class TNodesHolder>
+  class FileHolder : public cache::BaseFileHolder<TNodesHolder, cache::DataFileReader, FileReader>
+  {
+    typedef cache::DataFileReader reader_t;
+    typedef cache::BaseFileHolder<TNodesHolder, reader_t, FileReader> base_type;
+
+    typedef typename base_type::offset_map_t offset_map_t;
+
+    typedef typename base_type::user_id_t user_id_t;
+
+    template <class TElement, class ToDo> struct process_base
+    {
+    protected:
+      reader_t & m_reader;
+      ToDo & m_toDo;
+    public:
+      process_base(reader_t & reader, ToDo & toDo) : m_reader(reader), m_toDo(toDo) {}
+
+      bool operator() (uint64_t id)
+      {
+        TElement e;
+        if (m_reader.Read(id, e))
+          return m_toDo(id, e);
+        return false;
+      }
+    };
+
+    template <class ToDo> struct process_relation : public process_base<RelationElement, ToDo>
+    {
+      typedef process_base<RelationElement, ToDo> base_type;
+    public:
+      process_relation(reader_t & reader, ToDo & toDo) : base_type(reader, toDo) {}
+    };
+
+    template <class ToDo> struct process_relation_cached : public process_relation<ToDo>
+    {
+      typedef process_relation<ToDo> base_type;
+
+    public:
+      process_relation_cached(reader_t & rels, ToDo & toDo)
+      : base_type(rels, toDo) {}
+
+      bool operator() (uint64_t id)
+      {
+        return this->m_toDo(id, this->m_reader);
+      }
+    };
+
+  public:
+    FileHolder(TNodesHolder & holder, string const & dir) : base_type(holder, dir) {}
+
+    bool GetNode(uint64_t id, double & lat, double & lng)
+    {
+      return this->m_nodes.GetPoint(id, lat, lng);
+    }
+
+    bool GetWay(user_id_t id, WayElement & e)
+    {
+      return this->m_ways.Read(id, e);
+    }
+
+    template <class ToDo> void ForEachRelationByWay(user_id_t id, ToDo & toDo)
+    {
+      process_relation<ToDo> processor(this->m_relations, toDo);
+      this->m_ways2rel.for_each_ret(id, processor);
+    }
+
+    template <class ToDo> void ForEachRelationByNodeCached(user_id_t id, ToDo & toDo)
+    {
+      process_relation_cached<ToDo> processor(this->m_relations, toDo);
+      this->m_nodes2rel.for_each_ret(id, processor);
+    }
+
+    template <class ToDo> void ForEachRelationByWayCached(user_id_t id, ToDo & toDo)
+    {
+      process_relation_cached<ToDo> processor(this->m_relations, toDo);
+      this->m_ways2rel.for_each_ret(id, processor);
+    }
+    
+    void LoadIndex()
+    {
+      this->m_ways.LoadOffsets();
+      this->m_relations.LoadOffsets();
+      
+      this->m_nodes2rel.read_to_memory();
+      this->m_ways2rel.read_to_memory();
+    }
+  };
+} // namespace feature
+
+namespace data
+{
+
+  template <class TNodesHolder>
+  class FileHolder : public cache::BaseFileHolder<TNodesHolder, cache::DataFileWriter, FileWriter>
+  {
+    typedef cache::BaseFileHolder<TNodesHolder, cache::DataFileWriter, FileWriter> base_type;
+
+    typedef typename base_type::user_id_t user_id_t;
+
+    template <class TMap, class TVec>
+    void add_id2rel_vector(TMap & rMap, user_id_t relid, TVec const & v)
+    {
+      for (size_t i = 0; i < v.size(); ++i)
+        rMap.write(v[i].first, relid);
+    }
+
+  public:
+    FileHolder(TNodesHolder & nodes, string const & dir) : base_type(nodes, dir) {}
+
+    void AddNode(uint64_t id, double lat, double lng)
+    {
+      this->m_nodes.AddPoint(id, lat, lng);
+    }
+
+    void AddWay(user_id_t id, WayElement const & e)
+    {
+      this->m_ways.Write(id, e);
+    }
+
+    void AddRelation(user_id_t id, RelationElement const & e)
+    {
+      const string relationType = e.GetType();
+      if (relationType == "multipolygon" || relationType == "route" || relationType == "boundary")
+      {
+        this->m_relations.Write(id, e);
+
+        add_id2rel_vector(this->m_nodes2rel, id, e.nodes);
+        add_id2rel_vector(this->m_ways2rel, id, e.ways);
+      }
+    }
+
+    void SaveIndex()
+    {
+      this->m_ways.SaveOffsets();
+      this->m_relations.SaveOffsets();
+      
+      this->m_nodes2rel.flush_to_file();
+      this->m_ways2rel.flush_to_file();
+    }
+  };
+} // namespace data
+
+namespace
+{
+
+  class MainFeaturesEmitter
+  {
+    typedef WorldMapGenerator<feature::FeaturesCollector> WorldGenerator;
+    typedef CountryMapGenerator<feature::Polygonizer<feature::FeaturesCollector> > CountriesGenerator;
+    unique_ptr<CountriesGenerator> m_countries;
+    unique_ptr<WorldGenerator> m_world;
+
+    unique_ptr<CoastlineFeaturesGenerator> m_coasts;
+    unique_ptr<feature::FeaturesCollector> m_coastsHolder;
+
+    string m_srcCoastsFile;
+
+    template <class T1, class T2> class CombinedEmitter
+    {
+      T1 * m_p1;
+      T2 * m_p2;
+    public:
+      CombinedEmitter(T1 * p1, T2 * p2) : m_p1(p1), m_p2(p2) {}
+      void operator() (FeatureBuilder1 const & fb, uint64_t)
+      {
+        if (m_p1) (*m_p1)(fb);
+        if (m_p2) (*m_p2)(fb);
+      }
+    };
+
+    enum TypeIndex
+    {
+      NATURAL_COASTLINE,
+      NATURAL_LAND,
+      PLACE_ISLAND,
+      PLACE_ISLET,
+
+      TYPES_COUNT
+    };
+    uint32_t m_types[TYPES_COUNT];
+
+    inline uint32_t Type(TypeIndex i) const { return m_types[i]; }
+
+  public:
+    MainFeaturesEmitter(feature::GenerateInfo const & info)
+    {
+      Classificator const & c = classif();
+
+      char const * arr[][2] = {
+        { "natural", "coastline" },
+        { "natural", "land" },
+        { "place", "island" },
+        { "place", "islet" }
+      };
+      STATIC_ASSERT(ARRAY_SIZE(arr) == TYPES_COUNT);
+
+      for (size_t i = 0; i < ARRAY_SIZE(arr); ++i)
+        m_types[i] = c.GetTypeByPath(vector<string>(arr[i], arr[i] + 2));
+
+      m_srcCoastsFile = info.m_tmpDir + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix;
+
+      CHECK(!info.m_makeCoasts || !info.m_createWorld,
+            ("We can't do make_coasts and generate_world at the same time"));
+
+      if (!info.m_makeCoasts)
+      {
+        m_countries.reset(new CountriesGenerator(info));
+
+        if (info.m_emitCoasts)
+        {
+          m_coastsHolder.reset(new feature::FeaturesCollector(
+                                                     info.m_datFilePrefix + WORLD_COASTS_FILE_NAME + info.m_datFileSuffix));
+        }
+      }
+      else
+      {
+        // 4-10 - level range for cells
+        // 20000 - max points count per feature
+        m_coasts.reset(new CoastlineFeaturesGenerator(Type(NATURAL_COASTLINE), 4, 10, 20000));
+
+        m_coastsHolder.reset(new feature::FeaturesCollector(m_srcCoastsFile));
+      }
+
+      if (info.m_createWorld)
+        m_world.reset(new WorldGenerator(info));
+    }
+
+    void operator() (FeatureBuilder1 fb)
+    {
+      uint32_t const coastType = Type(NATURAL_COASTLINE);
+      bool const hasCoast = fb.HasType(coastType);
+
+      if (m_coasts)
+      {
+        if (hasCoast)
+        {
+          CHECK ( fb.GetGeomType() != feature::GEOM_POINT, () );
+
+          // leave only coastline type
+          fb.SetType(coastType);
+          (*m_coasts)(fb);
+        }
+      }
+      else
+      {
+        if (hasCoast)
+        {
+          fb.PopExactType(Type(NATURAL_LAND));
+          fb.PopExactType(coastType);
+        }
+        else if ((fb.HasType(Type(PLACE_ISLAND)) || fb.HasType(Type(PLACE_ISLET))) &&
+                 fb.GetGeomType() == feature::GEOM_AREA)
+        {
+          fb.AddType(Type(NATURAL_LAND));
+        }
+
+        if (fb.RemoveInvalidTypes())
+        {
+          if (m_world)
+            (*m_world)(fb);
+
+          if (m_countries)
+            (*m_countries)(fb);
+        }
+      }
+    }
+
+    /// @return false if coasts are not merged and FLAG_fail_on_coasts is set
+    bool Finish()
+    {
+      if (m_world)
+        m_world->DoMerge();
+
+      if (m_coasts)
+      {
+        // Check and stop if some coasts were not merged
+        if (!m_coasts->Finish())
+          return false;
+
+        size_t const count = m_coasts->GetCellsCount();
+        LOG(LINFO, ("Generating coastline polygons", count));
+
+        for (size_t i = 0; i < count; ++i)
+        {
+          vector<FeatureBuilder1> vecFb;
+          m_coasts->GetFeatures(i, vecFb);
+
+          for (size_t j = 0; j < vecFb.size(); ++j)
+            (*m_coastsHolder)(vecFb[j]);
+        }
+      }
+      else if (m_coastsHolder)
+      {
+        CombinedEmitter<feature::FeaturesCollector, CountriesGenerator>
+        emitter(m_coastsHolder.get(), m_countries.get());
+        feature::ForEachFromDatRawFormat(m_srcCoastsFile, emitter);
+      }
+      return true;
+    }
+    
+    inline void GetNames(vector<string> & names) const
+    {
+      if (m_countries)
+        names = m_countries->Parent().Names();
+      else
+        names.clear();
+    }
+  };
+} // namespace anonymous
+
+
+template <typename HolderT>
+void BuildIntermediateDataFromO5M(SourceReader & stream, HolderT & holder)
+{
+  O5mreader * reader;
+  O5mreaderRet rc;
+
+  if ((rc = o5mreader_open(&reader, stream.Handle())) != O5MREADER_RET_OK)
+  {
+    LOG(LCRITICAL, ("O5M Open error:", o5mreader_strerror(rc)));
+    exit(1);  
+  }
+
+  O5mreaderDataset ds;
+  O5mreaderIterateRet ret;
+  while ((ret = o5mreader_iterateDataSet(reader, &ds)) == O5MREADER_ITERATE_RET_NEXT)
+  {
+    O5mreaderIterateRet ret2;
+    switch (ds.type)
+    {
+      case O5MREADER_DS_NODE:
+      {
+        // Could do something with ds.id, ds.lon, ds.lat here, lon and lat are ints in 1E+7 * degree units
+        // convert to mercator
+        double const lat = MercatorBounds::LatToY(DECODE_O5M_COORD(ds.lat));
+        double const lng = MercatorBounds::LonToX(DECODE_O5M_COORD(ds.lon));
+        holder.AddNode(ds.id, lat, lng);
+      } break;
+
+      case O5MREADER_DS_WAY:
+      {
+        // store way
+        WayElement way(ds.id);
+        uint64_t nodeId = 0;
+        while ((ret2 = o5mreader_iterateNds(reader, &nodeId)) == O5MREADER_ITERATE_RET_NEXT)
+          way.nodes.push_back(nodeId);
+
+        if (way.IsValid())
+          holder.AddWay(ds.id, way);
+      } break;
+
+      case O5MREADER_DS_REL:
+      {
+        // store relation
+        RelationElement relation;
+        uint64_t refId = 0;
+        uint8_t type = 0;
+        char * role = nullptr;
+        while ((ret2 = o5mreader_iterateRefs(reader, &refId, &type, &role)) == O5MREADER_ITERATE_RET_NEXT)
+        {
+          // Could do something with refId (way or node or rel id depends on type), type and role
+          if (type == O5MREADER_DS_NODE)
+            relation.nodes.emplace_back(make_pair(refId, string(role)));
+          else if (type == O5MREADER_DS_WAY)
+            relation.ways.emplace_back(make_pair(refId, string(role)));
+          // we just ignore type == "relation"
+        }
+
+        char * key = nullptr;
+        char * val = nullptr;
+        while ((ret2 = o5mreader_iterateTags(reader, &key, &val)) == O5MREADER_ITERATE_RET_NEXT)
+          relation.tags.emplace(make_pair(string(key), string(val)));
+
+        if (relation.IsValid())
+          holder.AddRelation(ds.id, relation);
+      } break;
+    }
+  }
+}
+
+void BuildFeaturesFromO5M(SourceReader & stream, BaseOSMParser & parser)
+{
+  O5mreader * reader;
+  O5mreaderRet rc;
+
+  if ((rc = o5mreader_open(&reader, stream.Handle())) != O5MREADER_RET_OK)
+  {
+    LOG(LCRITICAL, ("O5M Open error:", o5mreader_strerror(rc)));
+    exit(1);
+  }
+
+  O5mreaderDataset ds;
+  O5mreaderIterateRet ret;
+  while ((ret = o5mreader_iterateDataSet(reader, &ds)) == O5MREADER_ITERATE_RET_NEXT)
+  {
+    XMLElement p;
+    p.id = ds.id;
+
+    O5mreaderIterateRet ret2;
+    char * key = nullptr;
+    char * val = nullptr;
+    switch (ds.type)
+    {
+      case O5MREADER_DS_NODE:
+      {
+        p.tagKey = XMLElement::ET_NODE;
+        p.lat = DECODE_O5M_COORD(ds.lat);
+        p.lng = DECODE_O5M_COORD(ds.lon);
+      } break;
+
+      case O5MREADER_DS_WAY:
+      {
+        p.tagKey = XMLElement::ET_WAY;
+        uint64_t nodeId = 0;
+        while ((ret2 = o5mreader_iterateNds(reader, &nodeId)) == O5MREADER_ITERATE_RET_NEXT)
+          p.AddND(nodeId);
+      } break;
+
+      case O5MREADER_DS_REL:
+      {
+        p.tagKey = XMLElement::ET_RELATION;
+        uint64_t refId = 0;
+        uint8_t type = 0;
+        char * role = nullptr;
+        while ((ret2 = o5mreader_iterateRefs(reader, &refId, &type, &role)) == O5MREADER_ITERATE_RET_NEXT)
+        {
+          string strType;
+          switch (type)
+          {
+            case O5MREADER_DS_NODE:
+              strType = "node";
+              break;
+            case O5MREADER_DS_WAY:
+              strType = "way";
+              break;
+            case O5MREADER_DS_REL:
+              strType = "relation";
+              break;
+
+            default:
+              break;
+          }
+          p.AddMEMBER(refId, strType, role);
+        }
+      } break;
+    }
+
+    while ((ret2 = o5mreader_iterateTags(reader, &key, &val)) == O5MREADER_ITERATE_RET_NEXT)
+      p.AddKV(key, val);
+    parser.EmitElement(&p);
+  }
+}
+
+
+///////////////////////////////////////////////////////////////////////////////////////////////////
+// Generate functions implementations.
+///////////////////////////////////////////////////////////////////////////////////////////////////
+
+
+template <class NodesHolderT>
+bool GenerateFeaturesImpl(feature::GenerateInfo & info, string const &osmFileType, string const & osmFileName = string())
+{
+  try
+  {
+    NodesHolderT nodes(info.m_tmpDir + NODES_FILE);
+
+    typedef feature::FileHolder<NodesHolderT> HolderT;
+    HolderT holder(nodes, info.m_tmpDir);
+    holder.LoadIndex();
+
+    MainFeaturesEmitter bucketer(info);
+    SecondPassParser<MainFeaturesEmitter, HolderT> parser(
+                  bucketer, holder,
+                  info.m_makeCoasts ? classif().GetCoastType() : 0,
+                  info.m_addressFile);
+
+    SourceReader reader(osmFileName);
+
+    if (osmFileType == "xml")
+    {
+      ParseXMLSequence(reader, parser);
+    }
+    else if (osmFileType == "o5m")
+    {
+      BuildFeaturesFromO5M(reader, parser);
+    }
+    else
+    {
+      LOG(LERROR, ("Unknown source type:", osmFileType));
+      return false;
+    }
+
+    parser.Finish();
+
+    // Stop if coasts are not merged and FLAG_fail_on_coasts is set
+    if (!bucketer.Finish())
+      return false;
+
+    bucketer.GetNames(info.m_bucketNames);
+  }
+  catch (Reader::Exception const & e)
+  {
+    LOG(LERROR, ("Error with file ", e.what()));
+    return false;
+  }
+
+  return true;
+}
+
+template <class TNodesHolder>
+bool GenerateIntermediateDataImpl(string const & dir, string const &osmFileType, string const & osmFileName = string())
+{
+  try
+  {
+    TNodesHolder nodes(dir + NODES_FILE);
+    typedef data::FileHolder<TNodesHolder> HolderT;
+    HolderT holder(nodes, dir);
+
+    SourceReader reader(osmFileName);
+
+    LOG(LINFO, ("Data sorce format:", osmFileType));
+
+    if (osmFileType == "xml")
+    {
+      FirstPassParser<HolderT> parser(holder);
+      ParseXMLSequence(reader, parser);
+    }
+    else if (osmFileType == "o5m")
+    {
+      BuildIntermediateDataFromO5M(reader, holder);
+    }
+    else
+    {
+      LOG(LERROR, ("Unknown source type:", osmFileType));
+      return false;
+    }
+
+    LOG(LINFO, ("Added points count = ", nodes.GetCount()));
+
+    holder.SaveIndex();
+  }
+  catch (Writer::Exception const & e)
+  {
+    LOG(LERROR, ("Error with file ", e.what()));
+    return false;
+  }
+  return true;
+}
+
+
+bool GenerateFeatures(feature::GenerateInfo & info, string const & nodeStorage, string const &osmFileType, string const & osmFileName)
+{
+  if (nodeStorage == "raw")
+    return GenerateFeaturesImpl<RawFileShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileType, osmFileName);
+  else if (nodeStorage == "map")
+    return GenerateFeaturesImpl<MapFileShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileType, osmFileName);
+  else if (nodeStorage == "mem")
+    return GenerateFeaturesImpl<RawMemShortPointStorage<BasePointStorage::MODE_READ>>(info, osmFileType, osmFileName);
+  else
+    CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage));
+  return false;
+}
+
+
+bool GenerateIntermediateData(string const & dir, string const & nodeStorage, string const &osmFileType, string const & osmFileName)
+{
+  if (nodeStorage == "raw")
+    return GenerateIntermediateDataImpl<RawFileShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileType, osmFileName);
+  else if (nodeStorage == "map")
+    return GenerateIntermediateDataImpl<MapFileShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileType, osmFileName);
+  else if (nodeStorage == "mem")
+    return GenerateIntermediateDataImpl<RawMemShortPointStorage<BasePointStorage::MODE_WRITE>>(dir, osmFileType, osmFileName);
+  else
+    CHECK(nodeStorage.empty(), ("Incorrect node_storage type:", nodeStorage));
+  return false;
+}
+
diff --git a/generator/osm_source.hpp b/generator/osm_source.hpp
new file mode 100644
index 0000000000..6a2d9f297e
--- /dev/null
+++ b/generator/osm_source.hpp
@@ -0,0 +1,11 @@
+#pragma once
+
+#include "generate_info.hpp"
+
+#include "../std/string.hpp"
+
+
+bool GenerateFeatures(feature::GenerateInfo & info, string const & nodeStorage, string const &osmFileType, string const & osmFileName);
+bool GenerateIntermediateData(string const & dir, string const & nodeStorage, string const &osmFileType, string const & osmFileName);
+
+
diff --git a/generator/point_storage.hpp b/generator/point_storage.hpp
index a3929d08d3..79d9a3afb5 100644
--- a/generator/point_storage.hpp
+++ b/generator/point_storage.hpp
@@ -1,8 +1,9 @@
 #pragma once
 
 #include "../coding/mmap_reader.hpp"
-#include <sqlite3.h>
 
+#include "../std/iostream.hpp"
+#include "../std/type_traits.hpp"
 
 /// Used to store all world nodes inside temporary index file.
 /// To find node by id, just calculate offset inside index file:
@@ -12,14 +13,14 @@ struct LatLon
   double lat;
   double lon;
 };
-STATIC_ASSERT(sizeof(LatLon) == 16);
+static_assert(sizeof(LatLon) == 16, "Invalid structure size");
 
 struct ShortLatLon
 {
   int32_t lat;
   int32_t lon;
 };
-STATIC_ASSERT(sizeof(ShortLatLon) == 8);
+static_assert(sizeof(ShortLatLon) == 8, "Invalid structure size");
 
 struct LatLonPos
 {
@@ -27,7 +28,7 @@ struct LatLonPos
   double lat;
   double lon;
 };
-STATIC_ASSERT(sizeof(LatLonPos) == 24);
+static_assert(sizeof(LatLonPos) == 24, "Invalid structure size");
 
 struct ShortLatLonPos
 {
@@ -35,7 +36,7 @@ struct ShortLatLonPos
   int32_t lat;
   int32_t lon;
 };
-STATIC_ASSERT(sizeof(ShortLatLonPos) == 16);
+static_assert(sizeof(ShortLatLonPos) == 16, "Invalid structure size");
 
 
 class BasePointStorage
@@ -53,7 +54,7 @@ protected:
 public:
   enum EStorageMode {MODE_READ = false, MODE_WRITE = true};
 
-  BasePointStorage(string const & name, size_t factor)
+  BasePointStorage(string const & name, size_t factor = 1000)
   {
     m_progress.Begin(name, factor);
   }
@@ -62,154 +63,21 @@ public:
 };
 
 template < BasePointStorage::EStorageMode ModeT >
-class SQLitePointStorage : public BasePointStorage
-{
-  sqlite3 *m_db;
-  sqlite3_stmt *m_prepared_statement;
-
-public:
-  SQLitePointStorage(string const & name) : BasePointStorage(name, 1000)
-  {
-    if( sqlite3_open((name+".sqlite").c_str(), &m_db) != SQLITE_OK ){
-      fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(m_db));
-      sqlite3_close(m_db);
-      exit(1);
-    }
-    InitStorage(EnableIf<ModeT>());
-  }
-
-  ~SQLitePointStorage()
-  {
-    DoneStorage(EnableIf<ModeT>());
-    sqlite3_finalize(m_prepared_statement);
-    sqlite3_close(m_db);
-  }
-
-  void InitStorage(EnableIf<MODE_WRITE>)
-  {
-    string create_table("drop table if exists points; drop index if exists points_idx; create table points(id integer PRIMARY KEY, ll blob) WITHOUT ROWID;");
-    if( sqlite3_exec(m_db, create_table.c_str(), NULL, NULL, NULL ) != SQLITE_OK ){
-      fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(m_db));
-      sqlite3_close(m_db);
-      exit(1);
-    }
-
-    char* errorMessage;
-    sqlite3_exec(m_db, "PRAGMA synchronous=OFF", NULL, NULL, &errorMessage);
-    sqlite3_exec(m_db, "PRAGMA count_changes=OFF", NULL, NULL, &errorMessage);
-    sqlite3_exec(m_db, "PRAGMA journal_mode=MEMORY", NULL, NULL, &errorMessage);
-    sqlite3_exec(m_db, "PRAGMA temp_store=MEMORY", NULL, NULL, &errorMessage);
-
-    string insert("insert into points(id, ll) values(?,?);");
-    if( sqlite3_prepare_v2(m_db, insert.c_str(), -1, &m_prepared_statement, NULL) != SQLITE_OK ){
-      fprintf(stderr, "Can't open database: %s\n", sqlite3_errmsg(m_db));
-      sqlite3_close(m_db);
-      exit(1);
-    }
-    if( sqlite3_exec(m_db, "BEGIN TRANSACTION", NULL, NULL, NULL ) != SQLITE_OK ){
-      fprintf(stderr, "Can't start transaction: %s\n", sqlite3_errmsg(m_db));
-      sqlite3_close(m_db);
-      exit(1);
-    }
-  }
-
-  void InitStorage(EnableIf<MODE_READ>)
-  {
-    string select("select ll from points where id=?;");
-    if( sqlite3_prepare_v2(m_db, select.c_str(), -1, &m_prepared_statement, NULL) != SQLITE_OK ){
-      fprintf(stderr, "failed sqlite3_prepare_v2: %s\n", sqlite3_errmsg(m_db));
-      sqlite3_close(m_db);
-      exit(1);
-    }
-  }
-
-  void DoneStorage(EnableIf<MODE_WRITE>)
-  {
-    if( sqlite3_exec(m_db, "COMMIT TRANSACTION", NULL, NULL, NULL ) != SQLITE_OK ){
-      fprintf(stderr, "Can't end transaction: %s\n", sqlite3_errmsg(m_db));
-      sqlite3_close(m_db);
-      exit(1);
-    }
-    if( sqlite3_exec(m_db, "create unique index points_idx on points(id);", NULL, NULL, NULL ) != SQLITE_OK ){
-      fprintf(stderr, "Can't end transaction: %s\n", sqlite3_errmsg(m_db));
-      sqlite3_close(m_db);
-      exit(1);
-    }
-  }
-
-  void DoneStorage(EnableIf<MODE_READ>) {}
-
-  void AddPoint(uint64_t id, double lat, double lng)
-  {
-    LatLon ll = {lat, lng};
-
-    if (sqlite3_bind_int64(m_prepared_statement, 1, id) != SQLITE_OK) {
-      cerr << "bind1 failed: " << sqlite3_errmsg(m_db) << endl;
-      exit(1);
-    }
-    if (sqlite3_bind_blob(m_prepared_statement, 2, &ll, sizeof(ll), SQLITE_STATIC) != SQLITE_OK) {
-      cerr << "bind2 failed: " << sqlite3_errmsg(m_db) << endl;
-      exit(1);
-    }
-
-    if (sqlite3_step(m_prepared_statement) != SQLITE_DONE) {
-      cerr << "execution failed: " << sqlite3_errmsg(m_db) << endl;
-      exit(1);
-    }
-
-    sqlite3_reset(m_prepared_statement);
-    m_progress.Inc();
-  }
-
-  bool GetPoint(uint64_t id, double & lat, double & lng) const
-  {
-    bool found = true;
-    if (sqlite3_bind_int64(m_prepared_statement, 1, id) != SQLITE_OK) {
-      cerr << "bind1 failed: " << sqlite3_errmsg(m_db) << endl;
-      exit(1);
-    }
-    int rc = sqlite3_step(m_prepared_statement);
-    if (rc == SQLITE_DONE)
-    {
-      found = false;
-    }
-    else if (rc == SQLITE_ROW)
-    {
-      const void * data;
-      data = sqlite3_column_blob (m_prepared_statement, 0);
-      LatLon const &ll = *((LatLon const *)data);
-      lat = ll.lat;
-      lng = ll.lon;
-    }
-    else
-    {
-      cerr << "execution failed: " << sqlite3_errmsg(m_db) << endl;
-      exit(1);
-    }
-
-    sqlite3_reset(m_prepared_statement);
-    return found;
-  }
-
-};
-
-template < BasePointStorage::EStorageMode ModeT >
 class RawFilePointStorage : public BasePointStorage
 {
-
 #ifdef OMIM_OS_WINDOWS
   typedef FileReader FileReaderT;
 #else
   typedef MmapReader FileReaderT;
 #endif
 
-  typename std::conditional<ModeT, FileWriter, FileReaderT>::type m_file;
+  typename conditional<ModeT, FileWriter, FileReaderT>::type m_file;
 
 public:
-  RawFilePointStorage(string const & name) : BasePointStorage(name, 1000), m_file(name) {}
+  RawFilePointStorage(string const & name) : BasePointStorage(name), m_file(name) {}
 
   template <bool T = (ModeT == BasePointStorage::MODE_WRITE)>
-  typename std::enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
+  typename enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
   {
     LatLon ll;
     ll.lat = lat;
@@ -221,16 +89,10 @@ public:
   }
 
   template <bool T = (ModeT == BasePointStorage::MODE_READ)>
-  typename std::enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
+  typename enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
   {
-    // I think, it's not good idea to write this ugly code.
-    // memcpy isn't to slow for that.
-    //#ifdef OMIM_OS_WINDOWS
     LatLon ll;
     m_file.Read(id * sizeof(ll), &ll, sizeof(ll));
-    //#else
-    //    LatLon const & ll = *reinterpret_cast<LatLon const *>(m_file.Data() + id * sizeof(ll));
-    //#endif
 
     // assume that valid coordinate is not (0, 0)
     if (ll.lat != 0.0 || ll.lon != 0.0)
@@ -251,25 +113,24 @@ public:
 template < BasePointStorage::EStorageMode ModeT >
 class RawFileShortPointStorage : public BasePointStorage
 {
-
 #ifdef OMIM_OS_WINDOWS
   typedef FileReader FileReaderT;
 #else
   typedef MmapReader FileReaderT;
 #endif
 
-  typename std::conditional<ModeT, FileWriter, FileReaderT>::type m_file;
+  typename conditional<ModeT, FileWriter, FileReaderT>::type m_file;
 
-  double const m_precision = 10000000;
+  constexpr static double const kValueOrder = 1E+7;
 
 public:
-  RawFileShortPointStorage(string const & name) : BasePointStorage(name, 1000), m_file(name) {}
+  RawFileShortPointStorage(string const & name) : BasePointStorage(name), m_file(name) {}
 
   template <bool T = (ModeT == BasePointStorage::MODE_WRITE)>
-  typename std::enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
+  typename enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
   {
-    int64_t const lat64 = lat * m_precision;
-    int64_t const lng64 = lng * m_precision;
+    int64_t const lat64 = lat * kValueOrder;
+    int64_t const lng64 = lng * kValueOrder;
 
     ShortLatLon ll;
     ll.lat = static_cast<int32_t>(lat64);
@@ -284,22 +145,16 @@ public:
   }
 
   template <bool T = (ModeT == BasePointStorage::MODE_READ)>
-  typename std::enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
+  typename enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
   {
-    // I think, it's not good idea to write this ugly code.
-    // memcpy isn't to slow for that.
-    //#ifdef OMIM_OS_WINDOWS
     ShortLatLon ll;
     m_file.Read(id * sizeof(ll), &ll, sizeof(ll));
-    //#else
-    //    LatLon const & ll = *reinterpret_cast<LatLon const *>(m_file.Data() + id * sizeof(ll));
-    //#endif
 
     // assume that valid coordinate is not (0, 0)
     if (ll.lat != 0.0 || ll.lon != 0.0)
     {
-      lat = static_cast<double>(ll.lat) / m_precision;
-      lng = static_cast<double>(ll.lon) / m_precision;
+      lat = static_cast<double>(ll.lat) / kValueOrder;
+      lng = static_cast<double>(ll.lon) / kValueOrder;
       return true;
     }
     else
@@ -314,15 +169,17 @@ public:
 template < BasePointStorage::EStorageMode ModeT >
 class RawMemShortPointStorage : public BasePointStorage
 {
-  typename std::conditional<ModeT, FileWriter, FileReader>::type m_file;
+  typename conditional<ModeT, FileWriter, FileReader>::type m_file;
 
-  double const m_precision = 10000000;
+  constexpr static double const kValueOrder = 1E+7;
 
-  vector<ShortLatLon> m_data;
+  typedef ShortLatLon LonLatT;
+
+  vector<LonLatT> m_data;
 
 public:
   RawMemShortPointStorage(string const & name)
-  : BasePointStorage(name, 1000)
+  : BasePointStorage(name)
   , m_file(name)
   , m_data((size_t)0xFFFFFFFF)
   {
@@ -338,21 +195,21 @@ public:
 
   void InitStorage(EnableIf<MODE_READ>)
   {
-    m_file.Read(0, m_data.data(), m_data.size());
+    m_file.Read(0, m_data.data(), m_data.size() * sizeof(LonLatT));
   }
 
   void DoneStorage(EnableIf<MODE_WRITE>)
   {
-    m_file.Write(m_data.data(), m_data.size());
+    m_file.Write(m_data.data(), m_data.size() * sizeof(LonLatT));
   }
 
   void DoneStorage(EnableIf<MODE_READ>) {}
 
   template <bool T = (ModeT == BasePointStorage::MODE_WRITE)>
-  typename std::enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
+  typename enable_if<T, void>::type AddPoint(uint64_t id, double lat, double lng)
   {
-    int64_t const lat64 = lat * m_precision;
-    int64_t const lng64 = lng * m_precision;
+    int64_t const lat64 = lat * kValueOrder;
+    int64_t const lng64 = lng * kValueOrder;
 
     ShortLatLon & ll = m_data[id];
     ll.lat = static_cast<int32_t>(lat64);
@@ -364,14 +221,14 @@ public:
   }
 
   template <bool T = (ModeT == BasePointStorage::MODE_READ)>
-  typename std::enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
+  typename enable_if<T, bool>::type GetPoint(uint64_t id, double & lat, double & lng) const
   {
     ShortLatLon const & ll = m_data[id];
     // assume that valid coordinate is not (0, 0)
     if (ll.lat != 0.0 || ll.lon != 0.0)
     {
-      lat = static_cast<double>(ll.lat) / m_precision;
-      lng = static_cast<double>(ll.lon) / m_precision;
+      lat = static_cast<double>(ll.lat) / kValueOrder;
+      lng = static_cast<double>(ll.lon) / kValueOrder;
       return true;
     }
     else
@@ -382,13 +239,10 @@ public:
   }
 };
 
-
 template < BasePointStorage::EStorageMode ModeT >
 class MapFilePointStorage : public BasePointStorage
 {
-  typename std::conditional<ModeT, FileWriter, FileReader>::type m_file;
-
-
+  typename conditional<ModeT, FileWriter, FileReader>::type m_file;
   typedef unordered_map<uint64_t, pair<double, double> > ContainerT;
   ContainerT m_map;
 
@@ -412,7 +266,7 @@ public:
       LatLonPos ll;
       m_file.Read(pos, &ll, sizeof(ll));
 
-      (void)m_map.insert(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
+      m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
 
       pos += sizeof(ll);
     }
@@ -434,27 +288,22 @@ public:
   bool GetPoint(uint64_t id, double & lat, double & lng) const
   {
     auto i = m_map.find(id);
-    if (i != m_map.end())
-    {
-      lat = i->second.first;
-      lng = i->second.second;
-      return true;
-    }
-    return false;
+    if (i == m_map.end())
+      return false;
+    lat = i->second.first;
+    lng = i->second.second;
+    return true;
   }
-
 };
 
 template < BasePointStorage::EStorageMode ModeT >
 class MapFileShortPointStorage : public BasePointStorage
 {
-  typename std::conditional<ModeT, FileWriter, FileReader>::type m_file;
-
-
+  typename conditional<ModeT, FileWriter, FileReader>::type m_file;
   typedef unordered_map<uint64_t, pair<int32_t, int32_t> > ContainerT;
   ContainerT m_map;
 
-  double const m_precision = 10000000;
+  constexpr static double const kValueOrder = 1E+7;
 
 public:
   MapFileShortPointStorage(string const & name) : BasePointStorage(name, 10000), m_file(name+".short")
@@ -476,7 +325,7 @@ public:
       ShortLatLonPos ll;
       m_file.Read(pos, &ll, sizeof(ll));
 
-      (void)m_map.insert(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
+      m_map.emplace(make_pair(ll.pos, make_pair(ll.lat, ll.lon)));
 
       pos += sizeof(ll);
     }
@@ -487,10 +336,8 @@ public:
   void AddPoint(uint64_t id, double lat, double lng)
   {
 
-    int64_t const lat64 = lat * m_precision;
-    int64_t const lng64 = lng * m_precision;
-
-
+    int64_t const lat64 = lat * kValueOrder;
+    int64_t const lng64 = lng * kValueOrder;
 
     ShortLatLonPos ll;
     ll.pos = id;
@@ -506,13 +353,11 @@ public:
   bool GetPoint(uint64_t id, double & lat, double & lng) const
   {
     auto i = m_map.find(id);
-    if (i != m_map.end())
-    {
-      lat = static_cast<double>(i->second.first) / m_precision;
-      lng = static_cast<double>(i->second.second) / m_precision;
-      return true;
-    }
-    return false;
+    if (i == m_map.end())
+      return false;
+    lat = static_cast<double>(i->second.first) / kValueOrder;
+    lng = static_cast<double>(i->second.second) / kValueOrder;
+    return true;
   }
   
 };
diff --git a/generator/xml_element.cpp b/generator/xml_element.cpp
index 6c792544eb..7ff351f333 100644
--- a/generator/xml_element.cpp
+++ b/generator/xml_element.cpp
@@ -18,6 +18,29 @@ void XMLElement::AddKV(string const & k, string const & v)
   e.parent = this;
 }
 
+void XMLElement::AddND(uint64_t ref)
+{
+  childs.push_back(XMLElement());
+  XMLElement & e = childs.back();
+
+  e.tagKey = ET_ND;
+  e.ref = ref;
+  e.parent = this;
+}
+
+void XMLElement::AddMEMBER(uint64_t ref, string const & type, string const & role)
+{
+  childs.push_back(XMLElement());
+  XMLElement & e = childs.back();
+
+  e.tagKey = ET_MEMBER;
+  e.ref = ref;
+  e.type = type;
+  e.role = role;
+  e.parent = this;
+}
+
+
 void BaseOSMParser::AddAttr(string const & key, string const & value)
 {
   if (m_current)
@@ -101,46 +124,3 @@ void BaseOSMParser::Pop(string const &)
     (*m_current) = XMLElement();
   }
 }
-
-namespace
-{
-  struct StdinReader
-  {
-    uint64_t Read(char * buffer, uint64_t bufferSize)
-    {
-      return fread(buffer, sizeof(char), bufferSize, stdin);
-    }
-  };
-
-  struct FileReader
-  {
-    FILE * m_file;
-
-    FileReader(string const & filename)
-    {
-      m_file = fopen(filename.c_str(), "rb");
-    }
-
-    ~FileReader()
-    {
-      fclose(m_file);
-    }
-
-    uint64_t Read(char * buffer, uint64_t bufferSize)
-    {
-      return fread(buffer, sizeof(char), bufferSize, m_file);
-    }
-  };
-}
-
-void ParseXMLFromStdIn(BaseOSMParser & parser)
-{
-  StdinReader reader;
-  (void)ParseXMLSequence(reader, parser);
-}
-
-void ParseXMLFromFile(BaseOSMParser & parser, string const & osmFileName)
-{
-  FileReader reader(osmFileName);
-  (void)ParseXMLSequence(reader, parser);
-}
diff --git a/generator/xml_element.hpp b/generator/xml_element.hpp
index cd0ade8991..d4b22fe4cc 100644
--- a/generator/xml_element.hpp
+++ b/generator/xml_element.hpp
@@ -34,6 +34,9 @@ struct XMLElement
   vector<XMLElement> childs;
 
   void AddKV(string const & k, string const & v);
+  void AddND(uint64_t ref);
+  void AddMEMBER(uint64_t ref, string const & type, string const & role);
+
 };
 
 class BaseOSMParser
@@ -53,10 +56,8 @@ public:
   void Pop(string const &);
   void CharData(string const &) {}
 
+  virtual void EmitElement(XMLElement * p) = 0;
+
 protected:
   bool MatchTag(string const & tagName, XMLElement::ETag & tagKey);
-  virtual void EmitElement(XMLElement * p) = 0;
 };
-
-void ParseXMLFromStdIn(BaseOSMParser & parser);
-void ParseXMLFromFile(BaseOSMParser & parser, string const & osmFileName);