diff options
author | Ulrich Germann <ugermann@inf.ed.ac.uk> | 2015-02-15 04:45:22 +0300 |
---|---|---|
committer | Ulrich Germann <ugermann@inf.ed.ac.uk> | 2015-02-15 04:45:22 +0300 |
commit | ccf44f39fbef38eab0b818cb74c517b68a27ae0b (patch) | |
tree | 70ed8bb02cf049ffa6373572f3ded4fed0b43711 /moses/TranslationModel | |
parent | e520a3d39e212110a6d390dace6f587b20e7e91a (diff) |
Code cleanup and reorganization. A few classes have been renamed to shorter names.
Diffstat (limited to 'moses/TranslationModel')
-rw-r--r-- | moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp | 279 | ||||
-rw-r--r-- | moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h | 90 |
2 files changed, 189 insertions, 180 deletions
diff --git a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp index ad7591a7b..5f42bdeef 100644 --- a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp +++ b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp @@ -1,5 +1,6 @@ -// $Id$ +// -*- c++ -*- // vim:tabstop=2 +// $Id$ /*********************************************************************** Moses - factored phrase-based language decoder Copyright (C) 2006 University of Edinburgh @@ -24,151 +25,155 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace Moses { -LexicalReorderingTableCompact::LexicalReorderingTableCompact( - const std::string& filePath, - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors) - : LexicalReorderingTable(f_factors, e_factors, c_factors), - m_inMemory(StaticData::Instance().UseMinlexrInMemory()), - m_numScoreComponent(6), m_multipleScoreTrees(true), - m_hash(10, 16), m_scoreTrees(1) -{ - Load(filePath); -} - -LexicalReorderingTableCompact::LexicalReorderingTableCompact( - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors) - : LexicalReorderingTable(f_factors, e_factors, c_factors), - m_inMemory(StaticData::Instance().UseMinlexrInMemory()), - m_numScoreComponent(6), m_multipleScoreTrees(true), - m_hash(10, 16), m_scoreTrees(1) -{ } - -LexicalReorderingTableCompact::~LexicalReorderingTableCompact() -{ - for(size_t i = 0; i < m_scoreTrees.size(); i++) - delete m_scoreTrees[i]; -} - -std::vector<float> LexicalReorderingTableCompact::GetScore(const Phrase& f, - const Phrase& e, - const Phrase& c) -{ - std::string key; - Scores scores; - - if(0 == c.GetSize()) - key = MakeKey(f, e, c); - else - for(size_t i = 0; i <= c.GetSize(); ++i) { - Phrase sub_c(c.GetSubString(WordsRange(i,c.GetSize()-1))); - key = MakeKey(f,e,sub_c); - } - - size_t index = m_hash[key]; - if(m_hash.GetSize() != index) { - std::string scoresString; - if(m_inMemory) - scoresString = m_scoresMemory[index]; - else - scoresString = m_scoresMapped[index]; - - BitWrapper<> bitStream(scoresString); - for(size_t i = 0; i < m_numScoreComponent; i++) - scores.push_back(m_scoreTrees[m_multipleScoreTrees ? i : 0]->Read(bitStream)); - - return scores; + LexicalReorderingTableCompact:: + LexicalReorderingTableCompact(const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) + : LexicalReorderingTable(f_factors, e_factors, c_factors) + , m_inMemory(StaticData::Instance().UseMinlexrInMemory()) + , m_numScoreComponent(6) + , m_multipleScoreTrees(true) + , m_hash(10, 16) + , m_scoreTrees(1) + { + Load(filePath); } - return Scores(); -} - -std::string LexicalReorderingTableCompact::MakeKey(const Phrase& f, - const Phrase& e, - const Phrase& c) const -{ - return MakeKey(Trim(f.GetStringRep(m_FactorsF)), - Trim(e.GetStringRep(m_FactorsE)), - Trim(c.GetStringRep(m_FactorsC))); -} + LexicalReorderingTableCompact:: + LexicalReorderingTableCompact(const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) + : LexicalReorderingTable(f_factors, e_factors, c_factors) + , m_inMemory(StaticData::Instance().UseMinlexrInMemory()) + , m_numScoreComponent(6) + , m_multipleScoreTrees(true) + , m_hash(10, 16) + , m_scoreTrees(1) + { } + + LexicalReorderingTableCompact:: + ~LexicalReorderingTableCompact() + { + for(size_t i = 0; i < m_scoreTrees.size(); i++) + delete m_scoreTrees[i]; + } -std::string LexicalReorderingTableCompact::MakeKey(const std::string& f, - const std::string& e, - const std::string& c) const -{ - std::string key; - if(!f.empty()) { - key += f; + std::vector<float> + LexicalReorderingTableCompact:: + GetScore(const Phrase& f, const Phrase& e, const Phrase& c) + { + std::string key; + Scores scores; + + if(0 == c.GetSize()) + key = MakeKey(f, e, c); + else + for(size_t i = 0; i <= c.GetSize(); ++i) + { + Phrase sub_c(c.GetSubString(WordsRange(i,c.GetSize()-1))); + key = MakeKey(f,e,sub_c); + } + + size_t index = m_hash[key]; + if(m_hash.GetSize() != index) + { + std::string scoresString; + if(m_inMemory) + scoresString = m_scoresMemory[index]; + else + scoresString = m_scoresMapped[index]; + + BitWrapper<> bitStream(scoresString); + for(size_t i = 0; i < m_numScoreComponent; i++) + scores.push_back(m_scoreTrees[m_multipleScoreTrees ? i : 0]->Read(bitStream)); + + return scores; + } + + return Scores(); } - if(!m_FactorsE.empty()) { - if(!key.empty()) { - key += " ||| "; - } - key += e; + + std::string + LexicalReorderingTableCompact:: + MakeKey(const Phrase& f, + const Phrase& e, + const Phrase& c) const + { + return MakeKey(Trim(f.GetStringRep(m_FactorsF)), + Trim(e.GetStringRep(m_FactorsE)), + Trim(c.GetStringRep(m_FactorsC))); } - if(!m_FactorsC.empty()) { - if(!key.empty()) { - key += " ||| "; - } - key += c; + + std::string + LexicalReorderingTableCompact:: + MakeKey(const std::string& f, + const std::string& e, + const std::string& c) const + { + std::string key; + if(!f.empty()) key += f; + if(!m_FactorsE.empty()) { if(!key.empty()) key += " ||| "; key += e; } + if(!m_FactorsC.empty()) { if(!key.empty()) key += " ||| "; key += c; } + key += " ||| "; + return key; } - key += " ||| "; - return key; -} -LexicalReorderingTable* LexicalReorderingTableCompact::CheckAndLoad( - const std::string& filePath, - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors) -{ + LexicalReorderingTable* + LexicalReorderingTableCompact:: + CheckAndLoad + (const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) + { #ifdef HAVE_CMPH - std::string minlexr = ".minlexr"; - // file name is specified without suffix - if(FileExists(filePath + minlexr)) { - //there exists a compact binary version use that - VERBOSE(2,"Using compact lexical reordering table" << std::endl); - return new LexicalReorderingTableCompact(filePath + minlexr, f_factors, e_factors, c_factors); - } - // file name is specified with suffix - if(filePath.substr(filePath.length() - minlexr.length(), minlexr.length()) == minlexr - && FileExists(filePath)) { - //there exists a compact binary version use that - VERBOSE(2,"Using compact lexical reordering table" << std::endl); - return new LexicalReorderingTableCompact(filePath, f_factors, e_factors, c_factors); - } + std::string minlexr = ".minlexr"; + // file name is specified without suffix + if(FileExists(filePath + minlexr)) { + //there exists a compact binary version use that + VERBOSE(2,"Using compact lexical reordering table" << std::endl); + return new LexicalReorderingTableCompact(filePath + minlexr, f_factors, e_factors, c_factors); + } + // file name is specified with suffix + if(filePath.substr(filePath.length() - minlexr.length(), minlexr.length()) == minlexr + && FileExists(filePath)) { + //there exists a compact binary version use that + VERBOSE(2,"Using compact lexical reordering table" << std::endl); + return new LexicalReorderingTableCompact(filePath, f_factors, e_factors, c_factors); + } #endif - return 0; -} - -void LexicalReorderingTableCompact::Load(std::string filePath) -{ - std::FILE* pFile = std::fopen(filePath.c_str(), "r"); - if(m_inMemory) - m_hash.Load(pFile); - else - m_hash.LoadIndex(pFile); - - size_t read = 0; - read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, pFile); - read += std::fread(&m_multipleScoreTrees, sizeof(m_multipleScoreTrees), 1, pFile); - - if(m_multipleScoreTrees) { - m_scoreTrees.resize(m_numScoreComponent); - for(size_t i = 0; i < m_numScoreComponent; i++) - m_scoreTrees[i] = new CanonicalHuffman<float>(pFile); - } else { - m_scoreTrees.resize(1); - m_scoreTrees[0] = new CanonicalHuffman<float>(pFile); + return 0; } - if(m_inMemory) - m_scoresMemory.load(pFile, false); - else - m_scoresMapped.load(pFile, true); -} - + void + LexicalReorderingTableCompact:: + Load(std::string filePath) + { + std::FILE* pFile = std::fopen(filePath.c_str(), "r"); + if(m_inMemory) + m_hash.Load(pFile); + else + m_hash.LoadIndex(pFile); + + size_t read = 0; + read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, pFile); + read += std::fread(&m_multipleScoreTrees, + sizeof(m_multipleScoreTrees), 1, pFile); + + if(m_multipleScoreTrees) { + m_scoreTrees.resize(m_numScoreComponent); + for(size_t i = 0; i < m_numScoreComponent; i++) + m_scoreTrees[i] = new CanonicalHuffman<float>(pFile); + } else { + m_scoreTrees.resize(1); + m_scoreTrees[0] = new CanonicalHuffman<float>(pFile); + } + + if(m_inMemory) + m_scoresMemory.load(pFile, false); + else + m_scoresMapped.load(pFile, true); + } + } diff --git a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h index 755398788..d54dd5273 100644 --- a/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h +++ b/moses/TranslationModel/CompactPT/LexicalReorderingTableCompact.h @@ -36,49 +36,53 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA namespace Moses { -class LexicalReorderingTableCompact: public LexicalReorderingTable -{ -private: - bool m_inMemory; - - size_t m_numScoreComponent; - bool m_multipleScoreTrees; - - BlockHashIndex m_hash; - - typedef CanonicalHuffman<float> ScoreTree; - std::vector<ScoreTree*> m_scoreTrees; - - StringVector<unsigned char, unsigned long, MmapAllocator> m_scoresMapped; - StringVector<unsigned char, unsigned long, std::allocator> m_scoresMemory; - - std::string MakeKey(const Phrase& f, const Phrase& e, const Phrase& c) const; - std::string MakeKey(const std::string& f, const std::string& e, const std::string& c) const; - -public: - LexicalReorderingTableCompact( - const std::string& filePath, - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors); - - LexicalReorderingTableCompact( - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors); - - virtual ~LexicalReorderingTableCompact(); - - virtual std::vector<float> GetScore(const Phrase& f, const Phrase& e, const Phrase& c); - - static LexicalReorderingTable* CheckAndLoad( - const std::string& filePath, - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors); - - void Load(std::string filePath); -}; + class LexicalReorderingTableCompact: + public LexicalReorderingTable + { + private: + bool m_inMemory; + + size_t m_numScoreComponent; + bool m_multipleScoreTrees; + + BlockHashIndex m_hash; + + typedef CanonicalHuffman<float> ScoreTree; + std::vector<ScoreTree*> m_scoreTrees; + + StringVector<unsigned char, unsigned long, MmapAllocator> m_scoresMapped; + StringVector<unsigned char, unsigned long, std::allocator> m_scoresMemory; + + std::string MakeKey(const Phrase& f, const Phrase& e, const Phrase& c) const; + std::string MakeKey(const std::string& f, const std::string& e, const std::string& c) const; + + public: + LexicalReorderingTableCompact(const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); + + LexicalReorderingTableCompact(const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); + + virtual + ~LexicalReorderingTableCompact(); + + virtual + std::vector<float> + GetScore(const Phrase& f, const Phrase& e, const Phrase& c); + + static + LexicalReorderingTable* + CheckAndLoad(const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); + + void + Load(std::string filePath); + }; } |