diff options
author | Hieu Hoang <s0565741@odin.inf.ed.ac.uk> | 2017-02-01 03:27:14 +0300 |
---|---|---|
committer | Hieu Hoang <s0565741@odin.inf.ed.ac.uk> | 2017-02-01 03:27:14 +0300 |
commit | 84b918b389e5a2d5e31cca993e1e53ff1354f1b1 (patch) | |
tree | b3891f5a5cfd55a41c9b75abe3b0040cc450a705 /moses2/TranslationModel | |
parent | adbb1c897a9ed019212dd7fc04e9a847e402e2d7 (diff) |
beautify
Diffstat (limited to 'moses2/TranslationModel')
36 files changed, 936 insertions, 1094 deletions
diff --git a/moses2/TranslationModel/CompactPT/BlockHashIndex.cpp b/moses2/TranslationModel/CompactPT/BlockHashIndex.cpp index 338a8e221..47f03626a 100644 --- a/moses2/TranslationModel/CompactPT/BlockHashIndex.cpp +++ b/moses2/TranslationModel/CompactPT/BlockHashIndex.cpp @@ -33,10 +33,10 @@ namespace Moses2 { #ifdef WITH_THREADS BlockHashIndex::BlockHashIndex(size_t orderBits, size_t fingerPrintBits, - size_t threadsNum) : - m_orderBits(orderBits), m_fingerPrintBits(fingerPrintBits), m_fileHandle(0), m_fileHandleStart( - 0), m_landmarks(true), m_size(0), m_lastSaved(-1), m_lastDropped(-1), m_numLoadedRanges( - 0), m_threadPool(threadsNum) + size_t threadsNum) : + m_orderBits(orderBits), m_fingerPrintBits(fingerPrintBits), m_fileHandle(0), m_fileHandleStart( + 0), m_landmarks(true), m_size(0), m_lastSaved(-1), m_lastDropped(-1), m_numLoadedRanges( + 0), m_threadPool(threadsNum) { #ifndef HAVE_CMPH std::cerr << "minphr: CMPH support not compiled in." << std::endl; @@ -45,9 +45,9 @@ BlockHashIndex::BlockHashIndex(size_t orderBits, size_t fingerPrintBits, } #else BlockHashIndex::BlockHashIndex(size_t orderBits, size_t fingerPrintBits) -: m_orderBits(orderBits), m_fingerPrintBits(fingerPrintBits), -m_fileHandle(0), m_fileHandleStart(0), m_size(0), -m_lastSaved(-1), m_lastDropped(-1), m_numLoadedRanges(0) + : m_orderBits(orderBits), m_fingerPrintBits(fingerPrintBits), + m_fileHandle(0), m_fileHandleStart(0), m_size(0), + m_lastSaved(-1), m_lastDropped(-1), m_numLoadedRanges(0) { #ifndef HAVE_CMPH std::cerr << "minphr: CMPH support not compiled in." << std::endl; @@ -60,11 +60,11 @@ BlockHashIndex::~BlockHashIndex() { #ifdef HAVE_CMPH for (std::vector<void*>::iterator it = m_hashes.begin(); it != m_hashes.end(); - it++) + it++) if (*it != 0) cmph_destroy((cmph_t*) *it); for (std::vector<PairedPackedArray<>*>::iterator it = m_arrays.begin(); - it != m_arrays.end(); it++) + it != m_arrays.end(); it++) if (*it != 0) delete *it; #endif } @@ -73,7 +73,7 @@ size_t BlockHashIndex::GetHash(const char* key) { std::string keyStr(key); size_t i = std::distance(m_landmarks.begin(), - std::upper_bound(m_landmarks.begin(), m_landmarks.end(), keyStr)) - 1; + std::upper_bound(m_landmarks.begin(), m_landmarks.end(), keyStr)) - 1; if (i == 0ul - 1) return GetSize(); @@ -99,14 +99,14 @@ size_t BlockHashIndex::GetHash(size_t i, const char* key) //LoadRange(i); #ifdef HAVE_CMPH size_t idx = cmph_search((cmph_t*) m_hashes[i], key, - (cmph_uint32) strlen(key)); + (cmph_uint32) strlen(key)); #else assert(0); size_t idx = 0; #endif std::pair<size_t, size_t> orderPrint = m_arrays[i]->Get(idx, m_orderBits, - m_fingerPrintBits); + m_fingerPrintBits); m_clocks[i] = clock(); if (GetFprint(key) == orderPrint.second) return orderPrint.first; @@ -229,7 +229,7 @@ size_t BlockHashIndex::FinalizeSave() size_t fileHandleStop = std::ftell(m_fileHandle); return fileHandleStop - m_fileHandleStart + sizeof(m_orderBits) - + sizeof(m_fingerPrintBits); + + sizeof(m_fingerPrintBits); } size_t BlockHashIndex::Save(std::FILE * mphf) @@ -262,7 +262,7 @@ size_t BlockHashIndex::LoadIndex(std::FILE* mphf) read += std::fread(&seekIndexSize, sizeof(size_t), 1, m_fileHandle); m_seekIndex.resize(seekIndexSize); read += std::fread(&m_seekIndex[0], sizeof(size_t), seekIndexSize, - m_fileHandle); + m_fileHandle); m_hashes.resize(seekIndexSize, 0); m_clocks.resize(seekIndexSize, 0); m_arrays.resize(seekIndexSize, 0); @@ -403,13 +403,13 @@ void* BlockHashIndex::vectorAdapter(std::vector<std::string>& v) } void* BlockHashIndex::vectorAdapter( - StringVector<unsigned, size_t, std::allocator>& sv) + StringVector<unsigned, size_t, std::allocator>& sv) { return (void*) CmphStringVectorAdapter(sv); } void* BlockHashIndex::vectorAdapter( - StringVector<unsigned, size_t, MmapAllocator>& sv) + StringVector<unsigned, size_t, MmapAllocator>& sv) { return (void*) CmphStringVectorAdapter(sv); } diff --git a/moses2/TranslationModel/CompactPT/BlockHashIndex.h b/moses2/TranslationModel/CompactPT/BlockHashIndex.h index b91ef8f6c..10c55601e 100644 --- a/moses2/TranslationModel/CompactPT/BlockHashIndex.h +++ b/moses2/TranslationModel/CompactPT/BlockHashIndex.h @@ -81,17 +81,14 @@ private: { public: HashTask(int id, BlockHashIndex& hash, Keys& keys) : - m_id(id), m_hash(hash), m_keys(new Keys(keys)) - { + m_id(id), m_hash(hash), m_keys(new Keys(keys)) { } - virtual void Run() - { + virtual void Run() { m_hash.CalcHash(m_id, *m_keys); } - virtual ~HashTask() - { + virtual ~HashTask() { delete m_keys; } @@ -108,7 +105,7 @@ private: public: #ifdef WITH_THREADS BlockHashIndex(size_t orderBits, size_t fingerPrintBits, - size_t threadsNum = 2); + size_t threadsNum = 2); #else BlockHashIndex(size_t orderBits, size_t fingerPrintBits); #endif @@ -147,8 +144,7 @@ public: void KeepNLastRanges(float ratio = 0.1, float tolerance = 0.1); template<typename Keys> - void AddRange(Keys &keys) - { + void AddRange(Keys &keys) { size_t current = m_landmarks.size(); if (m_landmarks.size() && m_landmarks.back().str() >= keys[0]) { @@ -171,7 +167,7 @@ public: #ifdef WITH_THREADS boost::shared_ptr<HashTask<Keys> > ht( - new HashTask<Keys>(current, *this, keys)); + new HashTask<Keys>(current, *this, keys)); m_threadPool.Submit(ht); #else CalcHash(current, keys); @@ -179,8 +175,7 @@ public: } template<typename Keys> - void CalcHash(size_t current, Keys &keys) - { + void CalcHash(size_t current, Keys &keys) { #ifdef HAVE_CMPH void* source = vectorAdapter(keys); CalcHash(current, source); diff --git a/moses2/TranslationModel/CompactPT/CanonicalHuffman.h b/moses2/TranslationModel/CompactPT/CanonicalHuffman.h index ffb6488c0..eb11c730a 100644 --- a/moses2/TranslationModel/CompactPT/CanonicalHuffman.h +++ b/moses2/TranslationModel/CompactPT/CanonicalHuffman.h @@ -43,24 +43,20 @@ private: typedef boost::unordered_map<Data, boost::dynamic_bitset<> > EncodeMap; EncodeMap m_encodeMap; - struct MinHeapSorter - { + struct MinHeapSorter { std::vector<size_t>& m_vec; MinHeapSorter(std::vector<size_t>& vec) : - m_vec(vec) - { + m_vec(vec) { } - bool operator()(size_t a, size_t b) - { + bool operator()(size_t a, size_t b) { return m_vec[a] > m_vec[b]; } }; template<class Iterator> - void CalcLengths(Iterator begin, Iterator end, std::vector<size_t>& lengths) - { + void CalcLengths(Iterator begin, Iterator end, std::vector<size_t>& lengths) { size_t n = std::distance(begin, end); std::vector<size_t> A(2 * n, 0); @@ -109,11 +105,10 @@ private: lengths[i] = A[i + n]; } - void CalcCodes(std::vector<size_t>& lengths) - { + void CalcCodes(std::vector<size_t>& lengths) { std::vector<size_t> numLength; for (std::vector<size_t>::iterator it = lengths.begin(); - it != lengths.end(); it++) { + it != lengths.end(); it++) { size_t length = *it; if (numLength.size() <= length) numLength.resize(length + 1, 0); numLength[length]++; @@ -139,7 +134,7 @@ private: size_t length = lengths[i]; size_t pos = m_lengthIndex[length] - + (nextCode[length] - m_firstCodes[length]); + + (nextCode[length] - m_firstCodes[length]); t_symbols[pos] = data; nextCode[length] = nextCode[length] + 1; @@ -148,13 +143,12 @@ private: m_symbols.swap(t_symbols); } - void CreateCodeMap() - { + void CreateCodeMap() { for (size_t l = 1; l < m_lengthIndex.size(); l++) { size_t intCode = m_firstCodes[l]; size_t num = ( - (l + 1 < m_lengthIndex.size()) ? - m_lengthIndex[l + 1] : m_symbols.size()) - m_lengthIndex[l]; + (l + 1 < m_lengthIndex.size()) ? + m_lengthIndex[l + 1] : m_symbols.size()) - m_lengthIndex[l]; for (size_t i = 0; i < num; i++) { Data data = m_symbols[m_lengthIndex[l] + i]; @@ -165,17 +159,15 @@ private: } } - const boost::dynamic_bitset<>& Encode(Data data) const - { + const boost::dynamic_bitset<>& Encode(Data data) const { typename EncodeMap::const_iterator it = m_encodeMap.find(data); UTIL_THROW_IF2(it == m_encodeMap.end(), - "Cannot find symbol in encoding map"); + "Cannot find symbol in encoding map"); return it->second; } template<class BitWrapper> - void PutCode(BitWrapper& bitWrapper, const boost::dynamic_bitset<>& code) - { + void PutCode(BitWrapper& bitWrapper, const boost::dynamic_bitset<>& code) { for (int j = code.size() - 1; j >= 0; j--) bitWrapper.Put(code[j]); } @@ -183,8 +175,7 @@ private: public: template<class Iterator> - CanonicalHuffman(Iterator begin, Iterator end, bool forEncoding = true) - { + CanonicalHuffman(Iterator begin, Iterator end, bool forEncoding = true) { std::vector<size_t> lengths; CalcLengths(begin, end, lengths); CalcCodes(lengths); @@ -192,22 +183,19 @@ public: if (forEncoding) CreateCodeMap(); } - CanonicalHuffman(std::FILE* pFile, bool forEncoding = false) - { + CanonicalHuffman(std::FILE* pFile, bool forEncoding = false) { Load(pFile); if (forEncoding) CreateCodeMap(); } template<class BitWrapper> - void Put(BitWrapper& bitWrapper, Data data) - { + void Put(BitWrapper& bitWrapper, Data data) { PutCode(bitWrapper, Encode(data)); } template<class BitWrapper> - Data Read(BitWrapper& bitWrapper) - { + Data Read(BitWrapper& bitWrapper) { if (bitWrapper.TellFromEnd()) { size_t intCode = bitWrapper.Read(); size_t len = 1; @@ -220,8 +208,7 @@ public: return Data(); } - size_t Load(std::FILE* pFile) - { + size_t Load(std::FILE* pFile) { size_t start = std::ftell(pFile); size_t read = 0; @@ -241,8 +228,7 @@ public: return std::ftell(pFile) - start; } - size_t Save(std::FILE* pFile) - { + size_t Save(std::FILE* pFile) { size_t start = std::ftell(pFile); size_t size = m_symbols.size(); @@ -277,24 +263,20 @@ private: public: BitWrapper(Container &data) : - m_data(data), m_iterator(m_data.begin()), m_currentValue(0), m_valueBits( - sizeof(typename Container::value_type) * 8), m_mask(1), m_bitPos(0) - { + m_data(data), m_iterator(m_data.begin()), m_currentValue(0), m_valueBits( + sizeof(typename Container::value_type) * 8), m_mask(1), m_bitPos(0) { } - bool Read() - { + bool Read() { if (m_bitPos % m_valueBits == 0) { if (m_iterator != m_data.end()) m_currentValue = *m_iterator++; - } - else m_currentValue = m_currentValue >> 1; + } else m_currentValue = m_currentValue >> 1; m_bitPos++; return (m_currentValue & m_mask); } - void Put(bool bit) - { + void Put(bool bit) { if (m_bitPos % m_valueBits == 0) m_data.push_back(0); if (bit) m_data[m_data.size() - 1] |= m_mask << (m_bitPos % m_valueBits); @@ -302,40 +284,34 @@ public: m_bitPos++; } - size_t Tell() - { + size_t Tell() { return m_bitPos; } - size_t TellFromEnd() - { + size_t TellFromEnd() { if (m_data.size() * m_valueBits < m_bitPos) return 0; return m_data.size() * m_valueBits - m_bitPos; } - void Seek(size_t bitPos) - { + void Seek(size_t bitPos) { m_bitPos = bitPos; m_iterator = m_data.begin() + int((m_bitPos - 1) / m_valueBits); m_currentValue = (*m_iterator) >> ((m_bitPos - 1) % m_valueBits); m_iterator++; } - void SeekFromEnd(size_t bitPosFromEnd) - { + void SeekFromEnd(size_t bitPosFromEnd) { size_t bitPos = m_data.size() * m_valueBits - bitPosFromEnd; Seek(bitPos); } - void Reset() - { + void Reset() { m_iterator = m_data.begin(); m_currentValue = 0; m_bitPos = 0; } - Container& GetContainer() - { + Container& GetContainer() { return m_data; } }; diff --git a/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.cpp b/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.cpp index a51dc5a45..8dc3ebde6 100644 --- a/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.cpp +++ b/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.cpp @@ -42,7 +42,7 @@ void CmphStringVectorAdapterRewind(void *data) cmph_io_adapter_t *CmphVectorAdapterNew(std::vector<std::string>& v) { cmph_io_adapter_t * key_source = (cmph_io_adapter_t *) malloc( - sizeof(cmph_io_adapter_t)); + sizeof(cmph_io_adapter_t)); cmph_vector_t * cmph_vector = (cmph_vector_t *) malloc(sizeof(cmph_vector_t)); assert(key_source); assert(cmph_vector); diff --git a/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.h b/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.h index 20d43a80c..8d23b4f41 100644 --- a/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.h +++ b/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.h @@ -33,18 +33,17 @@ namespace Moses2 { -typedef struct -{ +typedef struct { void *vector; cmph_uint32 position; } cmph_vector_t; template<typename ValueT, typename PosT, template<typename > class Allocator> cmph_io_adapter_t *CmphStringVectorAdapterNew( - StringVector<ValueT, PosT, Allocator>& sv) + StringVector<ValueT, PosT, Allocator>& sv) { cmph_io_adapter_t * key_source = (cmph_io_adapter_t *) malloc( - sizeof(cmph_io_adapter_t)); + sizeof(cmph_io_adapter_t)); cmph_vector_t * cmph_vector = (cmph_vector_t *) malloc(sizeof(cmph_vector_t)); assert(key_source); assert(cmph_vector); @@ -79,7 +78,7 @@ void CmphStringVectorAdapterRewind(void *data); template<typename ValueT, typename PosT, template<typename > class Allocator> cmph_io_adapter_t* CmphStringVectorAdapter( - StringVector<ValueT, PosT, Allocator>& sv) + StringVector<ValueT, PosT, Allocator>& sv) { cmph_io_adapter_t * key_source = CmphStringVectorAdapterNew(sv); diff --git a/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp b/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp index 1d32b9a6f..051116dec 100644 --- a/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp +++ b/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp @@ -32,23 +32,23 @@ namespace Moses2 bool LexicalReorderingTableCompact::s_inMemoryByDefault = false; LexicalReorderingTableCompact::LexicalReorderingTableCompact( - const std::string& filePath, const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors) : - LexicalReorderingTable(f_factors, e_factors, c_factors), m_inMemory( - s_inMemoryByDefault), m_numScoreComponent(6), m_multipleScoreTrees( - true), m_hash(10, 16), m_scoreTrees(1) + const std::string& filePath, const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) : + LexicalReorderingTable(f_factors, e_factors, c_factors), m_inMemory( + s_inMemoryByDefault), m_numScoreComponent(6), m_multipleScoreTrees( + true), m_hash(10, 16), m_scoreTrees(1) { Load(filePath); } LexicalReorderingTableCompact::LexicalReorderingTableCompact( - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors) : - LexicalReorderingTable(f_factors, e_factors, c_factors), m_inMemory( - s_inMemoryByDefault), m_numScoreComponent(6), m_multipleScoreTrees( - true), m_hash(10, 16), m_scoreTrees(1) + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) : + LexicalReorderingTable(f_factors, e_factors, c_factors), m_inMemory( + s_inMemoryByDefault), m_numScoreComponent(6), m_multipleScoreTrees( + true), m_hash(10, 16), m_scoreTrees(1) { } @@ -81,7 +81,7 @@ std::vector<float> LexicalReorderingTableCompact::GetScore(const Phrase<Moses2:: BitWrapper<> bitStream(scoresString); for (size_t i = 0; i < m_numScoreComponent; i++) scores.push_back( - m_scoreTrees[m_multipleScoreTrees ? i : 0]->Read(bitStream)); + m_scoreTrees[m_multipleScoreTrees ? i : 0]->Read(bitStream)); return scores; } @@ -93,7 +93,7 @@ std::string LexicalReorderingTableCompact::MakeKey(const Phrase<Moses2::Word>& f const Phrase<Moses2::Word>& e, const Phrase<Moses2::Word>& c) const { return MakeKey(Trim(f.GetString(m_FactorsF)), Trim(e.GetString(m_FactorsE)), - Trim(c.GetString(m_FactorsC))); + Trim(c.GetString(m_FactorsC))); } std::string LexicalReorderingTableCompact::MakeKey(const std::string& f, @@ -126,7 +126,7 @@ LexicalReorderingTableCompact::CheckAndLoad(const std::string& filePath, //there exists a compact binary version use that std::cerr << "Using compact lexical reordering table" << std::endl; return new LexicalReorderingTableCompact(filePath + minlexr, f_factors, - e_factors, c_factors); + e_factors, c_factors); } // file name is specified with suffix if (filePath.substr(filePath.length() - minlexr.length(), minlexr.length()) @@ -134,7 +134,7 @@ LexicalReorderingTableCompact::CheckAndLoad(const std::string& filePath, //there exists a compact binary version use that std::cerr << "Using compact lexical reordering table" << std::endl; return new LexicalReorderingTableCompact(filePath, f_factors, e_factors, - c_factors); + c_factors); } #endif return 0; @@ -152,16 +152,15 @@ void LexicalReorderingTableCompact::Load(std::string filePath) size_t read = 0; read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, - pFile); + pFile); read += std::fread(&m_multipleScoreTrees, sizeof(m_multipleScoreTrees), 1, - pFile); + pFile); if (m_multipleScoreTrees) { m_scoreTrees.resize(m_numScoreComponent); for (size_t i = 0; i < m_numScoreComponent; i++) m_scoreTrees[i] = new CanonicalHuffman<float>(pFile); - } - else { + } else { m_scoreTrees.resize(1); m_scoreTrees[0] = new CanonicalHuffman<float>(pFile); } diff --git a/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.h b/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.h index 90abf4197..cef6ae108 100644 --- a/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.h +++ b/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.h @@ -36,13 +36,11 @@ class LexicalReorderingTable { public: LexicalReorderingTable(const FactorList& f_factors, - const FactorList& e_factors, const FactorList& c_factors) : - m_FactorsF(f_factors), m_FactorsE(e_factors), m_FactorsC(c_factors) - { + const FactorList& e_factors, const FactorList& c_factors) : + m_FactorsF(f_factors), m_FactorsE(e_factors), m_FactorsC(c_factors) { } - virtual ~LexicalReorderingTable() - { + virtual ~LexicalReorderingTable() { } public: @@ -51,33 +49,27 @@ public: GetScore(const Phrase<Moses2::Word>& f, const Phrase<Moses2::Word>& e, const Phrase<Moses2::Word>& c) = 0; virtual - void InitializeForInput() - { + void InitializeForInput() { /* override for on-demand loading */ } ; virtual - void InitializeForInputPhrase(const Phrase<Moses2::Word>&) - { + void InitializeForInputPhrase(const Phrase<Moses2::Word>&) { } - const FactorList& GetFFactorMask() const - { + const FactorList& GetFFactorMask() const { return m_FactorsF; } - const FactorList& GetEFactorMask() const - { + const FactorList& GetEFactorMask() const { return m_FactorsE; } - const FactorList& GetCFactorMask() const - { + const FactorList& GetCFactorMask() const { return m_FactorsC; } virtual - void DbgDump(std::ostream* out) const - { + void DbgDump(std::ostream* out) const { *out << "Overwrite in subclass...\n"; } ; @@ -109,17 +101,17 @@ private: std::string MakeKey(const Phrase<Moses2::Word>& f, const Phrase<Moses2::Word>& e, const Phrase<Moses2::Word>& c) const; std::string MakeKey(const std::string& f, const std::string& e, - const std::string& c) const; + const std::string& c) const; public: LexicalReorderingTableCompact(const std::string& filePath, - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors); + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); LexicalReorderingTableCompact(const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors); + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); virtual ~LexicalReorderingTableCompact(); @@ -129,9 +121,9 @@ public: static LexicalReorderingTable* CheckAndLoad(const std::string& filePath, - const std::vector<FactorType>& f_factors, - const std::vector<FactorType>& e_factors, - const std::vector<FactorType>& c_factors); + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); void Load(std::string filePath); diff --git a/moses2/TranslationModel/CompactPT/ListCoders.h b/moses2/TranslationModel/CompactPT/ListCoders.h index 5a01274d9..540f50a59 100644 --- a/moses2/TranslationModel/CompactPT/ListCoders.h +++ b/moses2/TranslationModel/CompactPT/ListCoders.h @@ -33,8 +33,7 @@ class VarIntType { private: template<typename IntType, typename OutIt> - static void EncodeSymbol(IntType input, OutIt output) - { + static void EncodeSymbol(IntType input, OutIt output) { if (input == 0) { *output = 0; output++; @@ -56,8 +55,7 @@ private: ; template<typename InIt, typename IntType> - static void DecodeSymbol(InIt &it, InIt end, IntType &output) - { + static void DecodeSymbol(InIt &it, InIt end, IntType &output) { T msb = 1 << (sizeof(T) * 8 - 1); IntType shift = (sizeof(T) * 8 - 1); @@ -81,8 +79,7 @@ private: public: template<typename InIt, typename OutIt> - static void Encode(InIt it, InIt end, OutIt outIt) - { + static void Encode(InIt it, InIt end, OutIt outIt) { while (it != end) { EncodeSymbol(*it, outIt); it++; @@ -90,8 +87,7 @@ public: } template<typename InIt, typename OutIt> - static void Decode(InIt &it, InIt end, OutIt outIt) - { + static void Decode(InIt &it, InIt end, OutIt outIt) { while (it != end) { size_t output; DecodeSymbol(it, end, output); @@ -101,8 +97,7 @@ public: } template<typename InIt> - static size_t DecodeAndSum(InIt &it, InIt end, size_t num) - { + static size_t DecodeAndSum(InIt &it, InIt end, size_t num) { size_t sum = 0; size_t curr = 0; @@ -130,8 +125,7 @@ private: typedef unsigned int uint; template<typename InIt> - inline static void EncodeSymbol(uint &output, InIt it, InIt end) - { + inline static void EncodeSymbol(uint &output, InIt it, InIt end) { uint length = end - it; uint type = 0; @@ -182,8 +176,8 @@ private: uint i = 0; while (it != end) { UTIL_THROW_IF2(*it > 268435455, - "You are trying to encode " << *it - << " with Simple9. Cannot encode numbers larger than 268435455 (2^28-1)"); + "You are trying to encode " << *it + << " with Simple9. Cannot encode numbers larger than 268435455 (2^28-1)"); uint l = bitlength * (length - i - 1); output |= *it << l; @@ -193,8 +187,7 @@ private: } template<typename OutIt> - static inline void DecodeSymbol(uint input, OutIt outIt) - { + static inline void DecodeSymbol(uint input, OutIt outIt) { uint type = (input >> 28); uint bitlen = 0; @@ -258,8 +251,7 @@ private: outIt++; } - static inline size_t DecodeAndSumSymbol(uint input, size_t num, size_t &curr) - { + static inline size_t DecodeAndSumSymbol(uint input, size_t num, size_t &curr) { uint type = (input >> 28); uint bitlen = 0; @@ -327,8 +319,7 @@ private: public: template<typename InIt, typename OutIt> - static void Encode(InIt it, InIt end, OutIt outIt) - { + static void Encode(InIt it, InIt end, OutIt outIt) { uint parts[] = { 1, 2, 3, 4, 5, 7, 9, 14, 28 }; uint buffer[28]; @@ -367,8 +358,7 @@ public: } template<typename InIt, typename OutIt> - static void Decode(InIt &it, InIt end, OutIt outIt) - { + static void Decode(InIt &it, InIt end, OutIt outIt) { while (it != end) { DecodeSymbol(*it, outIt); it++; @@ -376,8 +366,7 @@ public: } template<typename InIt> - static size_t DecodeAndSum(InIt &it, InIt end, size_t num) - { + static size_t DecodeAndSum(InIt &it, InIt end, size_t num) { size_t sum = 0; size_t curr = 0; while (it != end && curr < num) { diff --git a/moses2/TranslationModel/CompactPT/MmapAllocator.h b/moses2/TranslationModel/CompactPT/MmapAllocator.h index 1e40d8d41..c92166da6 100644 --- a/moses2/TranslationModel/CompactPT/MmapAllocator.h +++ b/moses2/TranslationModel/CompactPT/MmapAllocator.h @@ -63,43 +63,37 @@ public: typedef std::ptrdiff_t difference_type; MmapAllocator() throw () : - m_file_ptr(std::tmpfile()), m_file_desc(fileno(m_file_ptr)), m_page_size( - util::SizePage()), m_map_size(0), m_data_ptr(0), m_data_offset(0), m_fixed( - false), m_count(new size_t(0)) - { + m_file_ptr(std::tmpfile()), m_file_desc(fileno(m_file_ptr)), m_page_size( + util::SizePage()), m_map_size(0), m_data_ptr(0), m_data_offset(0), m_fixed( + false), m_count(new size_t(0)) { } MmapAllocator(std::FILE* f_ptr) throw () : - m_file_ptr(f_ptr), m_file_desc(fileno(m_file_ptr)), m_page_size( - util::SizePage()), m_map_size(0), m_data_ptr(0), m_data_offset(0), m_fixed( - false), m_count(new size_t(0)) - { + m_file_ptr(f_ptr), m_file_desc(fileno(m_file_ptr)), m_page_size( + util::SizePage()), m_map_size(0), m_data_ptr(0), m_data_offset(0), m_fixed( + false), m_count(new size_t(0)) { } MmapAllocator(std::FILE* f_ptr, size_t data_offset) throw () : - m_file_ptr(f_ptr), m_file_desc(fileno(m_file_ptr)), m_page_size( - util::SizePage()), m_map_size(0), m_data_ptr(0), m_data_offset( - data_offset), m_fixed(true), m_count(new size_t(0)) - { + m_file_ptr(f_ptr), m_file_desc(fileno(m_file_ptr)), m_page_size( + util::SizePage()), m_map_size(0), m_data_ptr(0), m_data_offset( + data_offset), m_fixed(true), m_count(new size_t(0)) { } MmapAllocator(std::string fileName) throw () : - m_file_ptr(std::fopen(fileName.c_str(), "wb+")), m_file_desc( - fileno(m_file_ptr)), m_page_size(util::SizePage()), m_map_size(0), m_data_ptr( - 0), m_data_offset(0), m_fixed(false), m_count(new size_t(0)) - { + m_file_ptr(std::fopen(fileName.c_str(), "wb+")), m_file_desc( + fileno(m_file_ptr)), m_page_size(util::SizePage()), m_map_size(0), m_data_ptr( + 0), m_data_offset(0), m_fixed(false), m_count(new size_t(0)) { } MmapAllocator(const MmapAllocator& c) throw () : - m_file_ptr(c.m_file_ptr), m_file_desc(c.m_file_desc), m_page_size( - c.m_page_size), m_map_size(c.m_map_size), m_data_ptr(c.m_data_ptr), m_data_offset( - c.m_data_offset), m_fixed(c.m_fixed), m_count(c.m_count) - { + m_file_ptr(c.m_file_ptr), m_file_desc(c.m_file_desc), m_page_size( + c.m_page_size), m_map_size(c.m_map_size), m_data_ptr(c.m_data_ptr), m_data_offset( + c.m_data_offset), m_fixed(c.m_fixed), m_count(c.m_count) { (*m_count)++; } - ~MmapAllocator() throw () - { + ~MmapAllocator() throw () { if (m_data_ptr && *m_count == 0) { util::UnmapOrThrow(m_data_ptr, m_map_size); if (!m_fixed && std::ftell(m_file_ptr) != -1) std::fclose(m_file_ptr); @@ -108,28 +102,23 @@ public: } template<class U> - struct rebind - { + struct rebind { typedef MmapAllocator<U> other; }; - pointer address(reference value) const - { + pointer address(reference value) const { return &value; } - const_pointer address(const_reference value) const - { + const_pointer address(const_reference value) const { return &value; } - size_type max_size() const throw () - { + size_type max_size() const throw () { return std::numeric_limits<size_t>::max() / sizeof(value_type); } - pointer allocate(size_type num, const void* = 0) - { + pointer allocate(size_type num, const void* = 0) { m_map_size = num * sizeof(T); #if defined(_WIN32) || defined(_WIN64) @@ -142,57 +131,52 @@ public: size_t read = 0; read += ftruncate(m_file_desc, m_map_size); m_data_ptr = (char *) util::MapOrThrow(m_map_size, true, map_shared, - false, m_file_desc, 0); + false, m_file_desc, 0); return (pointer) m_data_ptr; - } - else { + } else { const size_t map_offset = (m_data_offset / m_page_size) * m_page_size; const size_t relative_offset = m_data_offset - map_offset; const size_t adjusted_map_size = m_map_size + relative_offset; m_data_ptr = (char *) util::MapOrThrow(adjusted_map_size, false, - map_shared, false, m_file_desc, map_offset); + map_shared, false, m_file_desc, map_offset); return (pointer) (m_data_ptr + relative_offset); } } - void deallocate(pointer p, size_type num) - { + void deallocate(pointer p, size_type num) { if (!m_fixed) { util::UnmapOrThrow(p, num * sizeof(T)); - } - else { + } else { const size_t map_offset = (m_data_offset / m_page_size) * m_page_size; const size_t relative_offset = m_data_offset - map_offset; const size_t adjusted_map_size = m_map_size + relative_offset; util::UnmapOrThrow((pointer) ((char*) p - relative_offset), - adjusted_map_size); + adjusted_map_size); } } - void construct(pointer p, const T& value) - { + void construct(pointer p, const T& value) { if (!m_fixed) new (p) value_type(value); } - void destroy(pointer p) - { + void destroy(pointer p) { if (!m_fixed) p->~T(); } template<class T1, class T2> friend bool operator==(const MmapAllocator<T1>&, - const MmapAllocator<T2>&) throw (); + const MmapAllocator<T2>&) throw (); template<class T1, class T2> friend bool operator!=(const MmapAllocator<T1>&, - const MmapAllocator<T2>&) throw (); + const MmapAllocator<T2>&) throw (); }; template<class T1, class T2> bool operator==(const MmapAllocator<T1>& a1, - const MmapAllocator<T2>& a2) throw () + const MmapAllocator<T2>& a2) throw () { bool equal = true; equal &= a1.m_file_ptr == a2.m_file_ptr; @@ -207,7 +191,7 @@ bool operator==(const MmapAllocator<T1>& a1, template<class T1, class T2> bool operator!=(const MmapAllocator<T1>& a1, - const MmapAllocator<T2>& a2) throw () + const MmapAllocator<T2>& a2) throw () { return !(a1 == a2); } diff --git a/moses2/TranslationModel/CompactPT/MonotonicVector.h b/moses2/TranslationModel/CompactPT/MonotonicVector.h index 586397db8..179354657 100644 --- a/moses2/TranslationModel/CompactPT/MonotonicVector.h +++ b/moses2/TranslationModel/CompactPT/MonotonicVector.h @@ -43,7 +43,7 @@ namespace Moses2 { template<typename PosT = size_t, typename NumT = size_t, PosT stepSize = 32, - template<typename > class Allocator = std::allocator> + template<typename > class Allocator = std::allocator> class MonotonicVector { private: @@ -62,17 +62,14 @@ public: typedef PosT value_type; MonotonicVector() : - m_size(0), m_last(0), m_final(false) - { + m_size(0), m_last(0), m_final(false) { } - size_t size() const - { + size_t size() const { return m_size + m_tempDiffs.size(); } - PosT at(size_t i) const - { + PosT at(size_t i) const { PosT s = stepSize; PosT j = m_anchors[i / s]; PosT r = i % s; @@ -83,23 +80,20 @@ public: k += VarInt32::DecodeAndSum(it, m_diffs.end(), 1); if (i < m_size) k += Simple9::DecodeAndSum(it, m_diffs.end(), r); else if (i < m_size + m_tempDiffs.size()) for (size_t l = 0; l < r; l++) - k += m_tempDiffs[l]; + k += m_tempDiffs[l]; return k; } - PosT operator[](PosT i) const - { + PosT operator[](PosT i) const { return at(i); } - PosT back() const - { + PosT back() const { return at(size() - 1); } - void push_back(PosT i) - { + void push_back(PosT i) { assert(m_final != true); if (m_anchors.size() == 0 && m_tempDiffs.size() == 0) { @@ -113,14 +107,13 @@ public: if (m_tempDiffs.size() == stepSize - 1) { Simple9::Encode(m_tempDiffs.begin(), m_tempDiffs.end(), - std::back_inserter(m_diffs)); + std::back_inserter(m_diffs)); m_anchors.push_back(m_diffs.size()); VarInt32::Encode(&i, &i + 1, std::back_inserter(m_diffs)); m_size += m_tempDiffs.size() + 1; m_tempDiffs.clear(); - } - else { + } else { PosT last = m_last; PosT diff = i - last; m_tempDiffs.push_back(diff); @@ -128,24 +121,21 @@ public: m_last = i; } - void commit() - { + void commit() { assert(m_final != true); Simple9::Encode(m_tempDiffs.begin(), m_tempDiffs.end(), - std::back_inserter(m_diffs)); + std::back_inserter(m_diffs)); m_size += m_tempDiffs.size(); m_tempDiffs.clear(); m_final = true; } - size_t usage() - { + size_t usage() { return m_diffs.size() * sizeof(unsigned int) - + m_anchors.size() * sizeof(NumT); + + m_anchors.size() * sizeof(NumT); } - size_t load(std::FILE* in, bool map = false) - { + size_t load(std::FILE* in, bool map = false) { size_t byteSize = 0; byteSize += fread(&m_final, sizeof(bool), 1, in) * sizeof(bool); @@ -160,8 +150,7 @@ public: template<typename ValueT> size_t loadVector(std::vector<ValueT, std::allocator<ValueT> >& v, - std::FILE* in, bool map = false) - { + std::FILE* in, bool map = false) { // Can only be read into memory. Mapping not possible with std:allocator. assert(map == false); @@ -178,8 +167,7 @@ public: template<typename ValueT> size_t loadVector(std::vector<ValueT, MmapAllocator<ValueT> >& v, - std::FILE* in, bool map = false) - { + std::FILE* in, bool map = false) { size_t byteSize = 0; size_t valSize; @@ -191,9 +179,8 @@ public: v.resize(valSize, 0); byteSize += std::fread(&v[0], sizeof(ValueT), valSize, in) - * sizeof(ValueT); - } - else { + * sizeof(ValueT); + } else { // Map it directly on specified region of file "in" starting at valPos // with length valSize * sizeof(ValueT). Mapped region cannot be resized. @@ -211,31 +198,29 @@ public: return byteSize; } - size_t save(std::FILE* out) - { + size_t save(std::FILE* out) { if (!m_final) commit(); bool byteSize = 0; byteSize += ThrowingFwrite(&m_final, sizeof(bool), 1, out) * sizeof(bool); byteSize += ThrowingFwrite(&m_size, sizeof(size_t), 1, out) - * sizeof(size_t); + * sizeof(size_t); byteSize += ThrowingFwrite(&m_last, sizeof(PosT), 1, out) * sizeof(PosT); size_t size = m_diffs.size(); byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t); byteSize += ThrowingFwrite(&m_diffs[0], sizeof(unsigned int), size, out) - * sizeof(unsigned int); + * sizeof(unsigned int); size = m_anchors.size(); byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t); byteSize += ThrowingFwrite(&m_anchors[0], sizeof(NumT), size, out) - * sizeof(NumT); + * sizeof(NumT); return byteSize; } - void swap(MonotonicVector<PosT, NumT, stepSize, Allocator> &mv) - { + void swap(MonotonicVector<PosT, NumT, stepSize, Allocator> &mv) { if (!m_final) commit(); m_diffs.swap(mv.m_diffs); diff --git a/moses2/TranslationModel/CompactPT/MurmurHash3.cpp b/moses2/TranslationModel/CompactPT/MurmurHash3.cpp index c3e567af6..988c1627f 100644 --- a/moses2/TranslationModel/CompactPT/MurmurHash3.cpp +++ b/moses2/TranslationModel/CompactPT/MurmurHash3.cpp @@ -151,7 +151,7 @@ void MurmurHash3_x86_32(const void * key, int len, uint32_t seed, void * out) //----------------------------------------------------------------------------- void MurmurHash3_x86_128(const void * key, const int len, uint32_t seed, - void * out) + void * out) { const uint8_t * data = (const uint8_t*) key; const int nblocks = len / 16; @@ -312,7 +312,7 @@ void MurmurHash3_x86_128(const void * key, const int len, uint32_t seed, //----------------------------------------------------------------------------- void MurmurHash3_x64_128(const void * key, const int len, const uint32_t seed, - void * out) + void * out) { const uint8_t * data = (const uint8_t*) key; const int nblocks = len / 16; diff --git a/moses2/TranslationModel/CompactPT/PackedArray.h b/moses2/TranslationModel/CompactPT/PackedArray.h index 409c3cca8..2da59a9f2 100644 --- a/moses2/TranslationModel/CompactPT/PackedArray.h +++ b/moses2/TranslationModel/CompactPT/PackedArray.h @@ -43,22 +43,19 @@ protected: D* m_storage; public: - PackedArray() - { + PackedArray() { m_size = 0; m_storageSize = 0; m_storage = new D[0]; } PackedArray(size_t size, size_t bits) : - m_size(size) - { + m_size(size) { m_storageSize = ceil(float(bits * size) / float(m_dataBits)); m_storage = new D[m_storageSize]; } - PackedArray(const PackedArray<T, D> &c) - { + PackedArray(const PackedArray<T, D> &c) { m_size = c.m_size; m_storageSize = c.m_storageSize; @@ -67,16 +64,14 @@ public: std::memcpy(m_storage, c.m_storage, m_storageSize * sizeof(D)); } - virtual ~PackedArray() - { + virtual ~PackedArray() { delete[] m_storage; m_size = 0; m_storageSize = 0; m_storage = 0; } - T Get(size_t i, size_t bits) const - { + T Get(size_t i, size_t bits) const { T out = 0; size_t bitstart = (i * bits); @@ -97,8 +92,7 @@ public: return out; } - void Set(size_t i, T v, size_t bits) - { + void Set(size_t i, T v, size_t bits) { size_t bitstart = (i * bits); size_t bitpos = bitstart; @@ -116,23 +110,19 @@ public: } } - virtual D*& GetStorage() - { + virtual D*& GetStorage() { return m_storage; } - virtual size_t GetStorageSize() const - { + virtual size_t GetStorageSize() const { return m_storageSize; } - virtual size_t Size() const - { + virtual size_t Size() const { return m_size; } - virtual size_t Load(std::FILE* in) - { + virtual size_t Load(std::FILE* in) { size_t a1 = std::ftell(in); size_t read = 0; @@ -146,8 +136,7 @@ public: return a2 - a1; } - virtual size_t Save(std::FILE* out) - { + virtual size_t Save(std::FILE* out) { size_t a1 = std::ftell(out); ThrowingFwrite(&m_size, sizeof(m_size), 1, out); @@ -170,31 +159,26 @@ class PairedPackedArray: public PackedArray<T, D> { public: PairedPackedArray() : - PackedArray<T, D>() - { + PackedArray<T, D>() { } PairedPackedArray(size_t size, size_t bits1, size_t bits2) : - PackedArray<T, D>(size, bits1 + bits2) - { + PackedArray<T, D>(size, bits1 + bits2) { } - void Set(size_t i, T a, T b, size_t bits1, size_t bits2) - { + void Set(size_t i, T a, T b, size_t bits1, size_t bits2) { T c = 0; c = a | (b << bits1); PackedArray<T, D>::Set(i, c, bits1 + bits2); } - void Set(size_t i, std::pair<T, T> p, size_t bits1, size_t bits2) - { + void Set(size_t i, std::pair<T, T> p, size_t bits1, size_t bits2) { T c = 0; c = p.second | (p.first << bits1); PackedArray<T, D>::Set(i, c); } - std::pair<T, T> Get(size_t i, size_t bits1, size_t bits2) - { + std::pair<T, T> Get(size_t i, size_t bits1, size_t bits2) { T v = PackedArray<T, D>::Get(i, bits1 + bits2); T a = v & ((1 << bits1) - 1); T b = v >> bits1; diff --git a/moses2/TranslationModel/CompactPT/PhraseDecoder.cpp b/moses2/TranslationModel/CompactPT/PhraseDecoder.cpp index 7860fed94..dc4369cfe 100644 --- a/moses2/TranslationModel/CompactPT/PhraseDecoder.cpp +++ b/moses2/TranslationModel/CompactPT/PhraseDecoder.cpp @@ -192,10 +192,10 @@ std::string PhraseDecoder::MakeSourceKey(std::string &source) } TargetPhraseVectorPtr PhraseDecoder::CreateTargetPhraseCollection( - const ManagerBase &mgr, - const Phrase<Word> &sourcePhrase, - bool topLevel, - bool eval) + const ManagerBase &mgr, + const Phrase<Word> &sourcePhrase, + bool topLevel, + bool eval) { // Not using TargetPhraseCollection avoiding "new" operator @@ -226,9 +226,9 @@ TargetPhraseVectorPtr PhraseDecoder::CreateTargetPhraseCollection( size_t sourcePhraseId = m_phraseDictionary.m_hash[MakeSourceKey(sourcePhraseString)]; /* cerr << "sourcePhraseString=" << sourcePhraseString << " " - << sourcePhraseId - << endl; - */ + << sourcePhraseId + << endl; + */ if(sourcePhraseId != m_phraseDictionary.m_hash.GetSize()) { // Retrieve compressed and encoded target phrase collection std::string encodedPhraseCollection; @@ -251,12 +251,12 @@ TargetPhraseVectorPtr PhraseDecoder::CreateTargetPhraseCollection( } TargetPhraseVectorPtr PhraseDecoder::DecodeCollection( - const ManagerBase &mgr, - TargetPhraseVectorPtr tpv, - BitWrapper<> &encodedBitStream, - const Phrase<Word> &sourcePhrase, - bool topLevel, - bool eval) + const ManagerBase &mgr, + TargetPhraseVectorPtr tpv, + BitWrapper<> &encodedBitStream, + const Phrase<Word> &sourcePhrase, + bool topLevel, + bool eval) { const System &system = mgr.system; FactorCollection &vocab = system.GetVocab(); diff --git a/moses2/TranslationModel/CompactPT/PhraseDecoder.h b/moses2/TranslationModel/CompactPT/PhraseDecoder.h index 79faa38a6..3113f4a0d 100644 --- a/moses2/TranslationModel/CompactPT/PhraseDecoder.h +++ b/moses2/TranslationModel/CompactPT/PhraseDecoder.h @@ -122,18 +122,18 @@ public: size_t Load(std::FILE* in); TargetPhraseVectorPtr CreateTargetPhraseCollection( - const ManagerBase &mgr, - const Phrase<Word> &sourcePhrase, - bool topLevel = false, - bool eval = true); + const ManagerBase &mgr, + const Phrase<Word> &sourcePhrase, + bool topLevel = false, + bool eval = true); TargetPhraseVectorPtr DecodeCollection( - const ManagerBase &mgr, - TargetPhraseVectorPtr tpv, - BitWrapper<> &encodedBitStream, - const Phrase<Word> &sourcePhrase, - bool topLevel, - bool eval); + const ManagerBase &mgr, + TargetPhraseVectorPtr tpv, + BitWrapper<> &encodedBitStream, + const Phrase<Word> &sourcePhrase, + bool topLevel, + bool eval); void PruneCache(); }; diff --git a/moses2/TranslationModel/CompactPT/PhraseTableCompact.cpp b/moses2/TranslationModel/CompactPT/PhraseTableCompact.cpp index 49244df1b..707a52f58 100644 --- a/moses2/TranslationModel/CompactPT/PhraseTableCompact.cpp +++ b/moses2/TranslationModel/CompactPT/PhraseTableCompact.cpp @@ -16,11 +16,11 @@ namespace Moses2 bool PhraseTableCompact::s_inMemoryByDefault = false; PhraseTableCompact::PhraseTableCompact(size_t startInd, const std::string &line) -:PhraseTable(startInd, line) -,m_inMemory(s_inMemoryByDefault) -,m_useAlignmentInfo(true) -,m_hash(10, 16) -,m_phraseDecoder(0) + :PhraseTable(startInd, line) + ,m_inMemory(s_inMemoryByDefault) + ,m_useAlignmentInfo(true) + ,m_hash(10, 16) + ,m_phraseDecoder(0) { ReadParameters(); } @@ -58,8 +58,7 @@ void PhraseTableCompact::Load(System &system) if(m_inMemory) { // Load target phrase collections into memory phraseSize = m_targetPhrasesMemory.load(pFile, false); - } - else { + } else { // Keep target phrase collections on disk phraseSize = m_targetPhrasesMapped.load(pFile, true); } @@ -72,8 +71,7 @@ void PhraseTableCompact::SetParameter(const std::string& key, const std::string& { if (key == "blah") { - } - else { + } else { PhraseTable::SetParameter(key, value); } } @@ -95,16 +93,16 @@ void PhraseTableCompact::Lookup(const Manager &mgr, InputPathsBase &inputPaths) InputPaths &inputPathsCast = static_cast<InputPaths&>(inputPaths); for (size_t i = 0; i < inputSize; ++i) { - for (size_t startPos = 0; startPos < inputSize; ++startPos) { - size_t endPos = startPos + i; - if (endPos >= inputSize) { - break; - } - InputPath *path = inputPathsCast.GetMatrix().GetValue(startPos, i); - //cerr << "path=" << path->Debug(mgr.system) << endl; - TargetPhrases *tps = Lookup(mgr, mgr.GetPool(), *path); - path->AddTargetPhrases(*this, tps); - } + for (size_t startPos = 0; startPos < inputSize; ++startPos) { + size_t endPos = startPos + i; + if (endPos >= inputSize) { + break; + } + InputPath *path = inputPathsCast.GetMatrix().GetValue(startPos, i); + //cerr << "path=" << path->Debug(mgr.system) << endl; + TargetPhrases *tps = Lookup(mgr, mgr.GetPool(), *path); + path->AddTargetPhrases(*this, tps); + } } } @@ -154,67 +152,67 @@ TargetPhrases *PhraseTableCompact::Lookup(const Manager &mgr, MemPool &pool, } const TargetPhraseImpl *PhraseTableCompact::CreateTargetPhrase( - const Manager &mgr, - const TPCompact &tpCompact, - const Phrase<Word> &sourcePhrase) const + const Manager &mgr, + const TPCompact &tpCompact, + const Phrase<Word> &sourcePhrase) const { - MemPool &pool = mgr.GetPool(); + MemPool &pool = mgr.GetPool(); - size_t size = tpCompact.words.size(); - TargetPhraseImpl *ret = new TargetPhraseImpl(pool, *this, mgr.system, size); + size_t size = tpCompact.words.size(); + TargetPhraseImpl *ret = new TargetPhraseImpl(pool, *this, mgr.system, size); - // words - for (size_t i = 0; i < size; ++i) { - const Word &compactWord = tpCompact.words[i]; - Word &tpWord = (*ret)[i]; - tpWord = compactWord; - } + // words + for (size_t i = 0; i < size; ++i) { + const Word &compactWord = tpCompact.words[i]; + Word &tpWord = (*ret)[i]; + tpWord = compactWord; + } - // scores - Scores &scores = ret->GetScores(); - scores.Assign(mgr.system, *this, tpCompact.scores); + // scores + Scores &scores = ret->GetScores(); + scores.Assign(mgr.system, *this, tpCompact.scores); - // align - ret->SetAlignTerm(tpCompact.alignment); + // align + ret->SetAlignTerm(tpCompact.alignment); - // score - mgr.system.featureFunctions.EvaluateInIsolation(pool, mgr.system, sourcePhrase, *ret); + // score + mgr.system.featureFunctions.EvaluateInIsolation(pool, mgr.system, sourcePhrase, *ret); - // Cache phrase pair for clean-up or retrieval with PREnc - //const_cast<PhraseDictionaryCompact*>(this)->CacheForCleanup(phraseColl); + // Cache phrase pair for clean-up or retrieval with PREnc + //const_cast<PhraseDictionaryCompact*>(this)->CacheForCleanup(phraseColl); - //cerr << "ret=" << ret->Debug(mgr.system) << endl; - return ret; + //cerr << "ret=" << ret->Debug(mgr.system) << endl; + return ret; } // scfg void PhraseTableCompact::InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const { UTIL_THROW2("Not implemented"); } void PhraseTableCompact::Lookup( - MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const + MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const { UTIL_THROW2("Not implemented"); } void PhraseTableCompact::LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const { UTIL_THROW2("Not implemented"); } diff --git a/moses2/TranslationModel/CompactPT/PhraseTableCompact.h b/moses2/TranslationModel/CompactPT/PhraseTableCompact.h index 84ea7e4b2..dced14c0c 100644 --- a/moses2/TranslationModel/CompactPT/PhraseTableCompact.h +++ b/moses2/TranslationModel/CompactPT/PhraseTableCompact.h @@ -18,22 +18,22 @@ public: virtual void CleanUpAfterSentenceProcessing() const; virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const; + InputPath &inputPath) const; // scfg virtual void InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const; + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; virtual void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const; virtual void Lookup( - MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const; + MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; protected: static bool s_inMemoryByDefault; @@ -49,19 +49,19 @@ protected: PhraseDecoder* m_phraseDecoder; const TargetPhraseImpl *CreateTargetPhrase( - const Manager &mgr, - const TPCompact &tpCompact, - const Phrase<Word> &sourcePhrase) const; + const Manager &mgr, + const TPCompact &tpCompact, + const Phrase<Word> &sourcePhrase) const; // SCFG virtual void LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; }; diff --git a/moses2/TranslationModel/CompactPT/StringVector.h b/moses2/TranslationModel/CompactPT/StringVector.h index 87d6388bf..0b2aa176f 100644 --- a/moses2/TranslationModel/CompactPT/StringVector.h +++ b/moses2/TranslationModel/CompactPT/StringVector.h @@ -53,13 +53,11 @@ public: const ValueIteratorT& begin() const; const ValueIteratorT& end() const; const std::string str() const; - operator const std::string() - { + operator const std::string() { return str(); } - size_t size() - { + size_t size() { return std::distance(m_begin, m_end); } @@ -75,7 +73,7 @@ public: // ********** StringVector ********** template<typename ValueT = unsigned char, typename PosT = unsigned int, - template<typename > class Allocator = std::allocator> + template<typename > class Allocator = std::allocator> class StringVector { protected: @@ -94,7 +92,7 @@ public: // ********** RangeIterator ********** class RangeIterator: public boost::iterator_facade<RangeIterator, range, - std::random_access_iterator_tag, range, PosT> + std::random_access_iterator_tag, range, PosT> { private: @@ -122,7 +120,7 @@ public: // ********** StringIterator ********** class StringIterator: public boost::iterator_facade<StringIterator, - std::string, std::random_access_iterator_tag, const std::string, PosT> + std::string, std::random_access_iterator_tag, const std::string, PosT> { private: @@ -152,13 +150,11 @@ public: StringVector(bool allocate = false); StringVector(Allocator<ValueT>& alloc); - virtual ~StringVector() - { + virtual ~StringVector() { delete m_charArray; } - void swap(StringVector<ValueT, PosT, Allocator> &c) - { + void swap(StringVector<ValueT, PosT, Allocator> &c) { m_positions.commit(); m_positions.swap(c.m_positions); m_charArray->swap(*c.m_charArray); @@ -184,8 +180,7 @@ public: const ValueT* begin(PosT i) const; const ValueT* end(PosT i) const; - void clear() - { + void clear() { m_charArray->clear(); m_sorted = true; m_positions = MonotonicVector<PosT, unsigned int, 32>(); @@ -203,8 +198,7 @@ public: PosT find(StringT &s) const; PosT find(const char* c) const; - virtual size_t load(std::FILE* in, bool memoryMapped = false) - { + virtual size_t load(std::FILE* in, bool memoryMapped = false) { size_t size = 0; m_memoryMapped = memoryMapped; @@ -216,8 +210,7 @@ public: } size_t loadCharArray(std::vector<ValueT, std::allocator<ValueT> >*& c, - std::FILE* in, bool map = false) - { + std::FILE* in, bool map = false) { // Can only be read into memory. Mapping not possible with std:allocator. assert(map == false); @@ -228,14 +221,13 @@ public: c = new std::vector<ValueT, std::allocator<ValueT> >(valSize, 0); byteSize += std::fread(&(*c)[0], sizeof(ValueT), valSize, in) - * sizeof(ValueT); + * sizeof(ValueT); return byteSize; } size_t loadCharArray(std::vector<ValueT, MmapAllocator<ValueT> >*& c, - std::FILE* in, bool map = false) - { + std::FILE* in, bool map = false) { size_t byteSize = 0; size_t valSize; @@ -246,9 +238,8 @@ public: // and map memory onto temporary file. Can be resized. c = new std::vector<ValueT, MmapAllocator<ValueT> >(valSize, 0); byteSize += std::fread(&(*c)[0], sizeof(ValueT), valSize, in) - * sizeof(ValueT); - } - else { + * sizeof(ValueT); + } else { // Map it directly on specified region of file "in" starting at valPos // with length valSize * sizeof(ValueT). Mapped region cannot be resized. @@ -263,16 +254,14 @@ public: return byteSize; } - size_t load(std::string filename, bool memoryMapped = false) - { + size_t load(std::string filename, bool memoryMapped = false) { std::FILE* pFile = fopen(filename.c_str(), "r"); size_t byteSize = load(pFile, memoryMapped); fclose(pFile); return byteSize; } - size_t save(std::FILE* out) - { + size_t save(std::FILE* out) { size_t byteSize = 0; byteSize += ThrowingFwrite(&m_sorted, sizeof(bool), 1, out) * sizeof(bool); @@ -280,15 +269,14 @@ public: size_t valSize = size2(); byteSize += ThrowingFwrite(&valSize, sizeof(size_t), 1, out) - * sizeof(size_t); + * sizeof(size_t); byteSize += ThrowingFwrite(&(*m_charArray)[0], sizeof(ValueT), valSize, out) - * sizeof(ValueT); + * sizeof(ValueT); return byteSize; } - size_t save(std::string filename) - { + size_t save(std::string filename) { std::FILE* pFile = fopen(filename.c_str(), "w"); size_t byteSize = save(pFile); fclose(pFile); @@ -304,7 +292,7 @@ public: template<typename ValueIteratorT> ValueIteratorRange<ValueIteratorT>::ValueIteratorRange(ValueIteratorT begin, ValueIteratorT end) : - m_begin(begin), m_end(end) + m_begin(begin), m_end(end) { } @@ -334,7 +322,7 @@ template<typename StringT> bool ValueIteratorRange<ValueIteratorT>::operator==(const StringT& o) const { if (std::distance(m_begin, m_end) == std::distance(o.begin(), o.end())) return std::equal( - m_begin, m_end, o.begin()); + m_begin, m_end, o.begin()); else return false; } @@ -349,7 +337,7 @@ template<typename StringT> bool ValueIteratorRange<ValueIteratorT>::operator<(const StringT &s2) const { return std::lexicographical_compare(m_begin, m_end, s2.begin(), s2.end(), - std::less<typename std::iterator_traits<ValueIteratorT>::value_type>()); + std::less<typename std::iterator_traits<ValueIteratorT>::value_type>()); } template<typename ValueIteratorT> @@ -362,8 +350,8 @@ template<typename StringT, typename ValueIteratorT> bool operator<(const StringT &s1, const ValueIteratorRange<ValueIteratorT> &s2) { return std::lexicographical_compare(s1.begin(), s1.end(), s2.begin(), - s2.end(), - std::less<typename std::iterator_traits<ValueIteratorT>::value_type>()); + s2.end(), + std::less<typename std::iterator_traits<ValueIteratorT>::value_type>()); } template<typename ValueIteratorT> @@ -371,7 +359,7 @@ bool operator<(const char* c, const ValueIteratorRange<ValueIteratorT> &s2) { size_t len = std::char_traits<char>::length(c); return std::lexicographical_compare(c, c + len, s2.begin(), s2.end(), - std::less<typename std::iterator_traits<ValueIteratorT>::value_type>()); + std::less<typename std::iterator_traits<ValueIteratorT>::value_type>()); } template<typename OStream, typename ValueIteratorT> @@ -387,15 +375,15 @@ OStream& operator<<(OStream &os, ValueIteratorRange<ValueIteratorT> cr) template<typename ValueT, typename PosT, template<typename > class Allocator> StringVector<ValueT, PosT, Allocator>::StringVector(bool allocate) : - m_sorted(true), m_memoryMapped(false), m_charArray( - allocate ? new std::vector<ValueT, Allocator<ValueT> >() : 0) + m_sorted(true), m_memoryMapped(false), m_charArray( + allocate ? new std::vector<ValueT, Allocator<ValueT> >() : 0) { } template<typename ValueT, typename PosT, template<typename > class Allocator> StringVector<ValueT, PosT, Allocator>::StringVector(Allocator<ValueT> &alloc) : - m_sorted(true), m_memoryMapped(false), m_charArray( - new std::vector<ValueT, Allocator<ValueT> >(alloc)) + m_sorted(true), m_memoryMapped(false), m_charArray( + new std::vector<ValueT, Allocator<ValueT> >(alloc)) { } @@ -428,12 +416,12 @@ template<typename Iterator> Iterator StringVector<ValueT, PosT, Allocator>::end() const { return Iterator(const_cast<StringVector<ValueT, PosT, Allocator>&>(*this), - size()); + size()); } template<typename ValueT, typename PosT, template<typename > class Allocator> typename StringVector<ValueT, PosT, Allocator>::iterator StringVector<ValueT, - PosT, Allocator>::begin() const + PosT, Allocator>::begin() const { return begin<iterator>(); } @@ -441,7 +429,7 @@ typename StringVector<ValueT, PosT, Allocator>::iterator StringVector<ValueT, template<typename ValueT, typename PosT, template<typename > class Allocator> typename StringVector<ValueT, PosT, Allocator>::iterator StringVector<ValueT, - PosT, Allocator>::end() const + PosT, Allocator>::end() const { return end<iterator>(); } @@ -467,21 +455,21 @@ PosT StringVector<ValueT, PosT, Allocator>::size2() const template<typename ValueT, typename PosT, template<typename > class Allocator> typename StringVector<ValueT, PosT, Allocator>::range StringVector<ValueT, PosT, - Allocator>::at(PosT i) const + Allocator>::at(PosT i) const { return range(begin(i), end(i)); } template<typename ValueT, typename PosT, template<typename > class Allocator> typename StringVector<ValueT, PosT, Allocator>::range StringVector<ValueT, PosT, - Allocator>::operator[](PosT i) const + Allocator>::operator[](PosT i) const { return at(i); } template<typename ValueT, typename PosT, template<typename > class Allocator> typename StringVector<ValueT, PosT, Allocator>::range StringVector<ValueT, PosT, - Allocator>::back() const + Allocator>::back() const { return at(size() - 1); } @@ -520,7 +508,7 @@ template<typename StringT> PosT StringVector<ValueT, PosT, Allocator>::find(StringT &s) const { if (m_sorted) return std::distance(begin(), - std::lower_bound(begin(), end(), s)); + std::lower_bound(begin(), end(), s)); return std::distance(begin(), std::find(begin(), end(), s)); } @@ -535,14 +523,14 @@ PosT StringVector<ValueT, PosT, Allocator>::find(const char* c) const template<typename ValueT, typename PosT, template<typename > class Allocator> StringVector<ValueT, PosT, Allocator>::RangeIterator::RangeIterator() : - m_index(0), m_container(0) + m_index(0), m_container(0) { } template<typename ValueT, typename PosT, template<typename > class Allocator> StringVector<ValueT, PosT, Allocator>::RangeIterator::RangeIterator( - StringVector<ValueT, PosT, Allocator> &sv, PosT index) : - m_index(index), m_container(&sv) + StringVector<ValueT, PosT, Allocator> &sv, PosT index) : + m_index(index), m_container(&sv) { } @@ -554,15 +542,15 @@ PosT StringVector<ValueT, PosT, Allocator>::RangeIterator::get_index() template<typename ValueT, typename PosT, template<typename > class Allocator> typename StringVector<ValueT, PosT, Allocator>::range StringVector<ValueT, PosT, - Allocator>::RangeIterator::dereference() const + Allocator>::RangeIterator::dereference() const { return typename StringVector<ValueT, PosT, Allocator>::range( - m_container->begin(m_index), m_container->end(m_index)); + m_container->begin(m_index), m_container->end(m_index)); } template<typename ValueT, typename PosT, template<typename > class Allocator> bool StringVector<ValueT, PosT, Allocator>::RangeIterator::equal( - StringVector<ValueT, PosT, Allocator>::RangeIterator const& other) const + StringVector<ValueT, PosT, Allocator>::RangeIterator const& other) const { return m_index == other.m_index && m_container == other.m_container; } @@ -587,7 +575,7 @@ void StringVector<ValueT, PosT, Allocator>::RangeIterator::advance(PosT n) template<typename ValueT, typename PosT, template<typename > class Allocator> PosT StringVector<ValueT, PosT, Allocator>::RangeIterator::distance_to( - StringVector<ValueT, PosT, Allocator>::RangeIterator const& other) const + StringVector<ValueT, PosT, Allocator>::RangeIterator const& other) const { return other.m_index - m_index; } @@ -596,14 +584,14 @@ PosT StringVector<ValueT, PosT, Allocator>::RangeIterator::distance_to( template<typename ValueT, typename PosT, template<typename > class Allocator> StringVector<ValueT, PosT, Allocator>::StringIterator::StringIterator() : - m_index(0), m_container(0) + m_index(0), m_container(0) { } template<typename ValueT, typename PosT, template<typename > class Allocator> StringVector<ValueT, PosT, Allocator>::StringIterator::StringIterator( - StringVector<ValueT, PosT, Allocator> &sv, PosT index) : - m_index(index), m_container(&sv) + StringVector<ValueT, PosT, Allocator> &sv, PosT index) : + m_index(index), m_container(&sv) { } @@ -617,12 +605,12 @@ template<typename ValueT, typename PosT, template<typename > class Allocator> const std::string StringVector<ValueT, PosT, Allocator>::StringIterator::dereference() const { return StringVector<ValueT, PosT, Allocator>::range( - m_container->begin(m_index), m_container->end(m_index)).str(); + m_container->begin(m_index), m_container->end(m_index)).str(); } template<typename ValueT, typename PosT, template<typename > class Allocator> bool StringVector<ValueT, PosT, Allocator>::StringIterator::equal( - StringVector<ValueT, PosT, Allocator>::StringIterator const& other) const + StringVector<ValueT, PosT, Allocator>::StringIterator const& other) const { return m_index == other.m_index && m_container == other.m_container; } @@ -647,7 +635,7 @@ void StringVector<ValueT, PosT, Allocator>::StringIterator::advance(PosT n) template<typename ValueT, typename PosT, template<typename > class Allocator> PosT StringVector<ValueT, PosT, Allocator>::StringIterator::distance_to( - StringVector<ValueT, PosT, Allocator>::StringIterator const& other) const + StringVector<ValueT, PosT, Allocator>::StringIterator const& other) const { return other.m_index - m_index; } diff --git a/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.h b/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.h index 3a9e6f170..75ab40c93 100644 --- a/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.h +++ b/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.h @@ -35,14 +35,12 @@ namespace Moses2 { typedef std::pair<size_t, size_t> AlignPointSizeT; -struct PhraseCompact : public std::vector<Word> -{ +struct PhraseCompact : public std::vector<Word> { public: PhraseCompact(const Phrase<Word> ©); }; -struct TPCompact -{ +struct TPCompact { std::vector<Word> words; std::set<AlignPointSizeT> alignment; std::vector<float> scores; diff --git a/moses2/TranslationModel/Memory/Node.h b/moses2/TranslationModel/Memory/Node.h index 97fa9618e..d5a6b8795 100644 --- a/moses2/TranslationModel/Memory/Node.h +++ b/moses2/TranslationModel/Memory/Node.h @@ -25,56 +25,50 @@ public: typedef boost::unordered_map<size_t, Node> Children; Node() - :m_targetPhrases(NULL) - ,m_unsortedTPS(NULL) + :m_targetPhrases(NULL) + ,m_unsortedTPS(NULL) {} ~Node() {} - void AddRule(const std::vector<FactorType> &factors, SP &source, TP *target) - { + void AddRule(const std::vector<FactorType> &factors, SP &source, TP *target) { AddRule(factors, source, target, 0); } - TPS *Find(const std::vector<FactorType> &factors, const SP &source, size_t pos = 0) const - { + TPS *Find(const std::vector<FactorType> &factors, const SP &source, size_t pos = 0) const { assert(source.GetSize()); if (pos == source.GetSize()) { return m_targetPhrases; - } - else { + } else { const WORD &word = source[pos]; //cerr << "word=" << word << endl; typename Children::const_iterator iter = m_children.find(word.hash(factors)); if (iter == m_children.end()) { return NULL; - } - else { + } else { const Node &child = iter->second; return child.Find(factors, source, pos + 1); } } } - const Node *Find(const std::vector<FactorType> &factors, const WORD &word) const - { + const Node *Find(const std::vector<FactorType> &factors, const WORD &word) const { typename Children::const_iterator iter = m_children.find(word.hash(factors)); if (iter == m_children.end()) { return NULL; - } - else { + } else { const Node &child = iter->second; return &child; } } - const TPS *GetTargetPhrases() const - { return m_targetPhrases; } + const TPS *GetTargetPhrases() const { + return m_targetPhrases; + } - void SortAndPrune(size_t tableLimit, MemPool &pool, System &system) - { - BOOST_FOREACH(typename Children::value_type &val, m_children){ + void SortAndPrune(size_t tableLimit, MemPool &pool, System &system) { + BOOST_FOREACH(typename Children::value_type &val, m_children) { Node &child = val.second; child.SortAndPrune(tableLimit, pool, system); } @@ -95,8 +89,9 @@ public: } } - const Children &GetChildren() const - { return m_children; } + const Children &GetChildren() const { + return m_children; + } void Debug(std::ostream &out, const System &system) const { BOOST_FOREACH(const typename Children::value_type &valPair, m_children) { @@ -110,8 +105,7 @@ protected: Phrase<WORD> *m_source; std::vector<TP*> *m_unsortedTPS; - Node &AddRule(const std::vector<FactorType> &factors, SP &source, TP *target, size_t pos) - { + Node &AddRule(const std::vector<FactorType> &factors, SP &source, TP *target, size_t pos) { if (pos == source.GetSize()) { if (m_unsortedTPS == NULL) { m_unsortedTPS = new std::vector<TP*>(); @@ -120,8 +114,7 @@ protected: m_unsortedTPS->push_back(target); return *this; - } - else { + } else { const WORD &word = source[pos]; Node &child = m_children[word.hash(factors)]; //std::cerr << "added " << word << " " << &child << " from " << this << std::endl; diff --git a/moses2/TranslationModel/Memory/PhraseTableMemory.cpp b/moses2/TranslationModel/Memory/PhraseTableMemory.cpp index 09eead137..9b231ebd6 100644 --- a/moses2/TranslationModel/Memory/PhraseTableMemory.cpp +++ b/moses2/TranslationModel/Memory/PhraseTableMemory.cpp @@ -37,9 +37,9 @@ namespace Moses2 //////////////////////////////////////////////////////////////////////// PhraseTableMemory::PhraseTableMemory(size_t startInd, const std::string &line) -:PhraseTable(startInd, line) -,m_rootPb(NULL) -,m_rootSCFG(NULL) + :PhraseTable(startInd, line) + ,m_rootPb(NULL) + ,m_rootSCFG(NULL) { ReadParameters(); } @@ -58,8 +58,7 @@ void PhraseTableMemory::Load(System &system) if (system.isPb) { m_rootPb = new PBNODE(); - } - else { + } else { m_rootSCFG = new SCFGNODE(); //cerr << "m_rootSCFG=" << m_rootSCFG << endl; } @@ -80,17 +79,17 @@ void PhraseTableMemory::Load(System &system) if (system.isPb) { PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, - toks[0]); + toks[0]); //cerr << "created soure" << endl; TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system, - toks[1]); + toks[1]); //cerr << "created target" << endl; target->GetScores().CreateFromString(toks[2], *this, system, true); //cerr << "created scores:" << *target << endl; if (toks.size() >= 4) { - //cerr << "alignstr=" << toks[3] << endl; - target->SetAlignmentInfo(toks[3]); + //cerr << "alignstr=" << toks[3] << endl; + target->SetAlignmentInfo(toks[3]); } // properties @@ -105,13 +104,12 @@ void PhraseTableMemory::Load(System &system) m_rootPb->AddRule(m_input, *source, target); //cerr << "target=" << target->Debug(system) << endl; - } - else { + } else { SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, - toks[0]); + toks[0]); //cerr << "created source:" << *source << endl; SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this, - system, toks[1]); + system, toks[1]); //cerr << "created target " << *target << " source=" << *source << endl; @@ -139,8 +137,7 @@ void PhraseTableMemory::Load(System &system) if (system.isPb) { m_rootPb->SortAndPrune(m_tableLimit, systemPool, system); //cerr << "root=" << &m_rootPb << endl; - } - else { + } else { m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system); //cerr << "root=" << &m_rootPb << endl; } @@ -162,9 +159,9 @@ TargetPhrases* PhraseTableMemory::Lookup(const Manager &mgr, MemPool &pool, } void PhraseTableMemory::InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const { size_t ptInd = GetPtInd(); ActiveChartEntryMem *chartEntry = new (pool.Allocate<ActiveChartEntryMem>()) ActiveChartEntryMem(pool, *m_rootSCFG); @@ -173,10 +170,10 @@ void PhraseTableMemory::InitActiveChart( } void PhraseTableMemory::Lookup(MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const { if (path.range.GetNumWordsCovered() > maxChartSpan) { return; @@ -213,13 +210,13 @@ void PhraseTableMemory::Lookup(MemPool &pool, } void PhraseTableMemory::LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const { const ActiveChartEntryMem &prevEntryCast = static_cast<const ActiveChartEntryMem&>(prevEntry); @@ -252,10 +249,10 @@ void PhraseTableMemory::LookupGivenNode( // there are some rules /* cerr << "outPath=" << outPath.range - << " bind=" << chartEntry->GetSymbolBind().Debug(mgr.system) - << " pt=" << GetPtInd() - << " tps=" << tps->Debug(mgr.system) << endl; - */ + << " bind=" << chartEntry->GetSymbolBind().Debug(mgr.system) + << " pt=" << GetPtInd() + << " tps=" << tps->Debug(mgr.system) << endl; + */ outPath.AddTargetPhrasesToPath(pool, mgr.system, *this, *tps, chartEntry->GetSymbolBind()); } diff --git a/moses2/TranslationModel/Memory/PhraseTableMemory.h b/moses2/TranslationModel/Memory/PhraseTableMemory.h index 035c7c9c5..07a47c7ff 100644 --- a/moses2/TranslationModel/Memory/PhraseTableMemory.h +++ b/moses2/TranslationModel/Memory/PhraseTableMemory.h @@ -33,16 +33,16 @@ class PhraseTableMemory: public PhraseTable const PhraseTableMemory::SCFGNODE &node; ActiveChartEntryMem(MemPool &pool, const PhraseTableMemory::SCFGNODE &vnode) - :Parent(pool) - ,node(vnode) + :Parent(pool) + ,node(vnode) {} ActiveChartEntryMem( - MemPool &pool, - const PhraseTableMemory::SCFGNODE &vnode, - const ActiveChartEntry &prevEntry) - :Parent(prevEntry) - ,node(vnode) + MemPool &pool, + const PhraseTableMemory::SCFGNODE &vnode, + const ActiveChartEntry &prevEntry) + :Parent(prevEntry) + ,node(vnode) {} }; @@ -53,31 +53,31 @@ public: virtual void Load(System &system); virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const; + InputPath &inputPath) const; virtual void InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const; + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; void Lookup(MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const; + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; protected: PBNODE *m_rootPb; SCFGNODE *m_rootSCFG; void LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; }; diff --git a/moses2/TranslationModel/PhraseTable.cpp b/moses2/TranslationModel/PhraseTable.cpp index c790147bb..fef6771d6 100644 --- a/moses2/TranslationModel/PhraseTable.cpp +++ b/moses2/TranslationModel/PhraseTable.cpp @@ -22,8 +22,8 @@ namespace Moses2 //////////////////////////////////////////////////////////////////////////// PhraseTable::PhraseTable(size_t startInd, const std::string &line) : - StatelessFeatureFunction(startInd, line), m_tableLimit(20) // default - , m_maxCacheSize(DEFAULT_MAX_TRANS_OPT_CACHE_SIZE) + StatelessFeatureFunction(startInd, line), m_tableLimit(20) // default + , m_maxCacheSize(DEFAULT_MAX_TRANS_OPT_CACHE_SIZE) { m_input.push_back(0); } @@ -37,20 +37,15 @@ void PhraseTable::SetParameter(const std::string& key, const std::string& value) { if (key == "cache-size") { m_maxCacheSize = Scan<size_t>(value); - } - else if (key == "path") { + } else if (key == "path") { m_path = value; - } - else if (key == "input-factor") { - m_input = Tokenize<FactorType>(value, ","); - } - else if (key == "output-factor") { - m_output = Tokenize<FactorType>(value, ","); - } - else if (key == "table-limit") { + } else if (key == "input-factor") { + m_input = Tokenize<FactorType>(value, ","); + } else if (key == "output-factor") { + m_output = Tokenize<FactorType>(value, ","); + } else if (key == "table-limit") { m_tableLimit = Scan<size_t>(value); - } - else { + } else { StatelessFeatureFunction::SetParameter(key, value); } } @@ -59,29 +54,27 @@ bool PhraseTable::SatisfyBackoff(const Manager &mgr, const InputPath &path) cons { const InputType &input = mgr.GetInput(); if ((mgr.system.options.input.xml_policy == XmlExclusive) - && input.XmlOverlap(path.range.GetStartPos(), path.range.GetEndPos())) { - return false; + && input.XmlOverlap(path.range.GetStartPos(), path.range.GetEndPos())) { + return false; + } + + //cerr << GetName() << "=" << GetPtInd() << "=" << decodeGraphBackoff << endl; + if (decodeGraphBackoff == 0) { + // always lookup + return true; + } else if (decodeGraphBackoff == -1) { + // lookup only if there's no existing rules + return path.GetNumRules() ? false : true; + } else if (path.range.GetNumWordsCovered() <= decodeGraphBackoff) { + return path.GetNumRules() ? false : true; } - //cerr << GetName() << "=" << GetPtInd() << "=" << decodeGraphBackoff << endl; - if (decodeGraphBackoff == 0) { - // always lookup - return true; - } - else if (decodeGraphBackoff == -1) { - // lookup only if there's no existing rules - return path.GetNumRules() ? false : true; - } - else if (path.range.GetNumWordsCovered() <= decodeGraphBackoff) { - return path.GetNumRules() ? false : true; - } - - return false; + return false; } void PhraseTable::Lookup(const Manager &mgr, InputPathsBase &inputPaths) const { - BOOST_FOREACH(InputPathBase *pathBase, inputPaths){ + BOOST_FOREACH(InputPathBase *pathBase, inputPaths) { InputPath *path = static_cast<InputPath*>(pathBase); //cerr << "path=" << path->range << " "; @@ -102,29 +95,29 @@ void PhraseTable::Lookup(const Manager &mgr, InputPathsBase &inputPaths) const } TargetPhrases *PhraseTable::Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const + InputPath &inputPath) const { UTIL_THROW2("Not implemented"); } void PhraseTable::EvaluateInIsolation(MemPool &pool, const System &system, - const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores, - SCORE &estimatedScore) const + const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const { } void PhraseTable::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source, - const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores, - SCORE &estimatedScore) const + const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores, + SCORE &estimatedScore) const { } // scfg void PhraseTable::LookupUnary(MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const { //cerr << "BEFORE LookupUnary" << path.Debug(mgr.system) << endl; size_t startPos = path.range.GetStartPos(); @@ -134,12 +127,12 @@ void PhraseTable::LookupUnary(MemPool &pool, } void PhraseTable::LookupNT( - MemPool &pool, - const SCFG::Manager &mgr, - const Moses2::Range &subPhraseRange, - const SCFG::InputPath &prevPath, - const SCFG::Stacks &stacks, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const { size_t endPos = outPath.range.GetEndPos(); @@ -161,13 +154,13 @@ void PhraseTable::LookupNT( } void PhraseTable::LookupGivenWord( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::InputPath &prevPath, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const { size_t ptInd = GetPtInd(); diff --git a/moses2/TranslationModel/PhraseTable.h b/moses2/TranslationModel/PhraseTable.h index 9237f5ba6..ef40c06a4 100644 --- a/moses2/TranslationModel/PhraseTable.h +++ b/moses2/TranslationModel/PhraseTable.h @@ -42,43 +42,45 @@ public: virtual void SetParameter(const std::string& key, const std::string& value); virtual void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const; virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const; + InputPath &inputPath) const; - void SetPtInd(size_t ind) - { m_ptInd = ind; } + void SetPtInd(size_t ind) { + m_ptInd = ind; + } - size_t GetPtInd() const - { return m_ptInd; } + size_t GetPtInd() const { + return m_ptInd; + } bool SatisfyBackoff(const Manager &mgr, const InputPath &path) const; virtual void EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source, - const TargetPhraseImpl &targetPhrase, Scores &scores, - SCORE &estimatedScore) const; + const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const; virtual void EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source, - const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores, - SCORE &estimatedScore) const; + const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores, + SCORE &estimatedScore) const; // scfg virtual void InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const = 0; + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const = 0; virtual void Lookup( - MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const = 0; + MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const = 0; virtual void LookupUnary(MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const; + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; protected: std::string m_path; @@ -89,38 +91,37 @@ protected: // cache size_t m_maxCacheSize; // 0 = no caching - struct CacheCollEntry2 - { + struct CacheCollEntry2 { TargetPhrases *tpsPtr; clock_t clock; }; // scfg virtual void LookupNT( - MemPool &pool, - const SCFG::Manager &mgr, - const Moses2::Range &subPhraseRange, - const SCFG::InputPath &prevPath, - const SCFG::Stacks &stacks, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const; virtual void LookupGivenWord( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::InputPath &prevPath, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; virtual void LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const = 0; + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const = 0; }; diff --git a/moses2/TranslationModel/ProbingPT/ProbingPT.cpp b/moses2/TranslationModel/ProbingPT/ProbingPT.cpp index 1f22f45be..d405bf026 100644 --- a/moses2/TranslationModel/ProbingPT/ProbingPT.cpp +++ b/moses2/TranslationModel/ProbingPT/ProbingPT.cpp @@ -30,17 +30,17 @@ using namespace std; namespace Moses2 { ProbingPT::ActiveChartEntryProbing::ActiveChartEntryProbing( - MemPool &pool, - const ActiveChartEntryProbing &prevEntry) -:Parent(prevEntry) -,m_key(prevEntry.m_key) + MemPool &pool, + const ActiveChartEntryProbing &prevEntry) + :Parent(prevEntry) + ,m_key(prevEntry.m_key) {} void ProbingPT::ActiveChartEntryProbing::AddSymbolBindElement( - const Range &range, - const SCFG::Word &word, - const Moses2::Hypotheses *hypos, - const Moses2::PhraseTable &pt) + const Range &range, + const SCFG::Word &word, + const Moses2::Hypotheses *hypos, + const Moses2::PhraseTable &pt) { const ProbingPT &probingPt = static_cast<const ProbingPT&>(pt); std::pair<bool, uint64_t> key = GetKey(word, probingPt); @@ -68,8 +68,8 @@ std::pair<bool, uint64_t> ProbingPT::ActiveChartEntryProbing::GetKey(const SCFG: //////////////////////////////////////////////////////////////////////////// ProbingPT::ProbingPT(size_t startInd, const std::string &line) -:PhraseTable(startInd, line) -,load_method(util::POPULATE_OR_READ) + :PhraseTable(startInd, line) + ,load_method(util::POPULATE_OR_READ) { ReadParameters(); } @@ -89,10 +89,10 @@ void ProbingPT::Load(System &system) // source vocab const std::map<uint64_t, std::string> &sourceVocab = - m_engine->getSourceVocab(); + m_engine->getSourceVocab(); std::map<uint64_t, std::string>::const_iterator iterSource; for (iterSource = sourceVocab.begin(); iterSource != sourceVocab.end(); - ++iterSource) { + ++iterSource) { string wordStr = iterSource->second; bool isNT; //cerr << "wordStr=" << wordStr << endl; @@ -142,29 +142,23 @@ void ProbingPT::Load(System &system) void ProbingPT::SetParameter(const std::string& key, const std::string& value) { - if (key == "load") { + if (key == "load") { if (value == "lazy") { load_method = util::LAZY; - } - else if (value == "populate_or_lazy") { + } else if (value == "populate_or_lazy") { load_method = util::POPULATE_OR_LAZY; - } - else if (value == "populate_or_read" || value == "populate") { + } else if (value == "populate_or_read" || value == "populate") { load_method = util::POPULATE_OR_READ; - } - else if (value == "read") { + } else if (value == "read") { load_method = util::READ; - } - else if (value == "parallel_read") { + } else if (value == "parallel_read") { load_method = util::PARALLEL_READ; + } else { + UTIL_THROW2("load method not supported" << value); } - else { - UTIL_THROW2("load method not supported" << value); - } - } - else { - PhraseTable::SetParameter(key, value); - } + } else { + PhraseTable::SetParameter(key, value); + } } void ProbingPT::CreateAlignmentMap(System &system, const std::string path) @@ -191,19 +185,19 @@ void ProbingPT::CreateAlignmentMap(System &system, const std::string path) void ProbingPT::Lookup(const Manager &mgr, InputPathsBase &inputPaths) const { - BOOST_FOREACH(InputPathBase *pathBase, inputPaths){ - InputPath *path = static_cast<InputPath*>(pathBase); - - if (SatisfyBackoff(mgr, *path)) { - TargetPhrases *tpsPtr; - tpsPtr = Lookup(mgr, mgr.GetPool(), *path); - path->AddTargetPhrases(*this, tpsPtr); - } + BOOST_FOREACH(InputPathBase *pathBase, inputPaths) { + InputPath *path = static_cast<InputPath*>(pathBase); + + if (SatisfyBackoff(mgr, *path)) { + TargetPhrases *tpsPtr; + tpsPtr = Lookup(mgr, mgr.GetPool(), *path); + path->AddTargetPhrases(*this, tpsPtr); + } } } TargetPhrases* ProbingPT::Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const + InputPath &inputPath) const { /* if (inputPath.prefixPath && inputPath.prefixPath->GetTargetPhrases(*this) == NULL) { @@ -234,7 +228,7 @@ TargetPhrases* ProbingPT::Lookup(const Manager &mgr, MemPool &pool, // query pt TargetPhrases *tps = CreateTargetPhrases(pool, mgr.system, sourcePhrase, - keyStruct.second); + keyStruct.second); return tps; } @@ -251,8 +245,7 @@ std::pair<bool, uint64_t> ProbingPT::GetKey(const Phrase<Moses2::Word> &sourcePh if (!ret.first) { // source phrase contains a word unknown in the pt. // We know immediately there's no translation for it - } - else { + } else { ret.second = m_engine->getKey(probingSource, sourceSize); } @@ -296,16 +289,16 @@ TargetPhrases *ProbingPT::CreateTargetPhrases(MemPool &pool, } TargetPhraseImpl *ProbingPT::CreateTargetPhrase( - MemPool &pool, - const System &system, - const char *&offset) const + MemPool &pool, + const System &system, + const char *&offset) const { TargetPhraseInfo *tpInfo = (TargetPhraseInfo*) offset; size_t numRealWords = tpInfo->numWords / m_output.size(); TargetPhraseImpl *tp = - new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, *this, - system, numRealWords); + new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, *this, + system, numRealWords); offset += sizeof(TargetPhraseInfo); @@ -322,8 +315,7 @@ TargetPhraseImpl *ProbingPT::CreateTargetPhrase( if (m_engine->num_lex_scores) { tp->scoreProperties = scores + m_engine->num_scores; } - } - else { + } else { // log score 1st SCORE logScores[totalNumScores]; for (size_t i = 0; i < totalNumScores; ++i) { @@ -344,20 +336,20 @@ TargetPhraseImpl *ProbingPT::CreateTargetPhrase( // words for (size_t targetPos = 0; targetPos < numRealWords; ++targetPos) { - for (size_t i = 0; i < m_output.size(); ++i) { - FactorType factorType = m_output[i]; + for (size_t i = 0; i < m_output.size(); ++i) { + FactorType factorType = m_output[i]; - uint32_t *probingId = (uint32_t*) offset; + uint32_t *probingId = (uint32_t*) offset; - const std::pair<bool, const Factor *> *factorPair = GetTargetFactor(*probingId); - assert(factorPair); - assert(!factorPair->first); + const std::pair<bool, const Factor *> *factorPair = GetTargetFactor(*probingId); + assert(factorPair); + assert(!factorPair->first); - Word &word = (*tp)[targetPos]; - word[factorType] = factorPair->second; + Word &word = (*tp)[targetPos]; + word[factorType] = factorPair->second; - offset += sizeof(uint32_t); - } + offset += sizeof(uint32_t); + } } // align @@ -372,7 +364,7 @@ TargetPhraseImpl *ProbingPT::CreateTargetPhrase( } void ProbingPT::GetSourceProbingIds(const Phrase<Moses2::Word> &sourcePhrase, - bool &ok, uint64_t probingSource[]) const + bool &ok, uint64_t probingSource[]) const { size_t size = sourcePhrase.GetSize(); @@ -382,8 +374,7 @@ void ProbingPT::GetSourceProbingIds(const Phrase<Moses2::Word> &sourcePhrase, if (probingId == m_unkId) { ok = false; return; - } - else { + } else { probingSource[i] = probingId; } } @@ -396,14 +387,14 @@ uint64_t ProbingPT::GetSourceProbingId(const Word &word) const uint64_t ret = 0; for (size_t i = 0; i < m_input.size(); ++i) { - FactorType factorType = m_input[i]; - const Factor *factor = word[factorType]; - - size_t factorId = factor->GetId(); - if (factorId >= m_sourceVocab.size()) { - return m_unkId; - } - ret += m_sourceVocab[factorId]; + FactorType factorType = m_input[i]; + const Factor *factor = word[factorType]; + + size_t factorId = factor->GetId(); + if (factorId >= m_sourceVocab.size()) { + return m_unkId; + } + ret += m_sourceVocab[factorId]; } return ret; @@ -435,12 +426,12 @@ void ProbingPT::CreateCache(System &system) //cerr << "line=" << line << endl; if (system.isPb) { - PhraseImpl *sourcePhrase = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, toks[2]); + PhraseImpl *sourcePhrase = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, toks[2]); - /* + /* std::pair<bool, uint64_t> retStruct = GetKey(*sourcePhrase); if (!retStruct.first) { - UTIL_THROW2("Unknown cache entry"); + UTIL_THROW2("Unknown cache entry"); } cerr << "key=" << retStruct.second << " " << key << endl; */ @@ -448,9 +439,8 @@ void ProbingPT::CreateCache(System &system) assert(tps); m_cachePb[key] = tps; - } - else { - // SCFG + } else { + // SCFG SCFG::PhraseImpl *sourcePhrase = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, toks[2], false); //cerr << "sourcePhrase=" << sourcePhrase->Debug(system) << endl; @@ -473,8 +463,7 @@ void ProbingPT::ReformatWord(System &system, std::string &wordStr, bool &isNT) isNT = false; if (system.isPb) { return; - } - else { + } else { isNT = (wordStr[0] == '[' && wordStr[wordStr.size() - 1] == ']'); //cerr << "nt=" << nt << endl; @@ -482,8 +471,7 @@ void ProbingPT::ReformatWord(System &system, std::string &wordStr, bool &isNT) size_t startPos = wordStr.find("]["); if (startPos == string::npos) { startPos = 1; - } - else { + } else { startPos += 2; } @@ -494,9 +482,9 @@ void ProbingPT::ReformatWord(System &system, std::string &wordStr, bool &isNT) } void ProbingPT::InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const { //cerr << "InitActiveChart=" << path.Debug(cerr, mgr.system) << endl; size_t ptInd = GetPtInd(); @@ -505,10 +493,10 @@ void ProbingPT::InitActiveChart( } void ProbingPT::Lookup(MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const { //cerr << "Lookup=" << endl; if (path.range.GetNumWordsCovered() > maxChartSpan) { @@ -546,13 +534,13 @@ void ProbingPT::Lookup(MemPool &pool, } void ProbingPT::LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const { const ActiveChartEntryProbing &prevEntryCast = static_cast<const ActiveChartEntryProbing&>(prevEntry); @@ -568,7 +556,7 @@ void ProbingPT::LookupGivenNode( // check in cache CacheSCFG::const_iterator iter = m_cacheSCFG.find(key.second); if (iter != m_cacheSCFG.end()) { - //cerr << "FOUND IN CACHE " << key.second << " " << sourcePhrase.Debug(mgr.system) << endl; + //cerr << "FOUND IN CACHE " << key.second << " " << sourcePhrase.Debug(mgr.system) << endl; SCFG::TargetPhrases *tps = iter->second; ActiveChartEntryProbing *chartEntry = new (pool.Allocate<ActiveChartEntryProbing>()) ActiveChartEntryProbing(pool, prevEntryCast); @@ -581,42 +569,41 @@ void ProbingPT::LookupGivenNode( outPath.AddActiveChartEntry(ptInd, chartEntry); outPath.AddTargetPhrasesToPath(pool, mgr.system, *this, *tps, chartEntry->GetSymbolBind()); - } - else { - // not in cache. Lookup - std::pair<bool, SCFG::TargetPhrases*> tpsPair = CreateTargetPhrasesSCFG(pool, mgr.system, sourcePhrase, key.second); - assert(tpsPair.first && tpsPair.second); - - if (tpsPair.first) { - // new entries - ActiveChartEntryProbing *chartEntry = new (pool.Allocate<ActiveChartEntryProbing>()) ActiveChartEntryProbing(pool, prevEntryCast); - //cerr << "AFTER chartEntry" << endl; - - chartEntry->AddSymbolBindElement(subPhraseRange, wordSought, hypos, *this); - //cerr << "AFTER AddSymbolBindElement" << endl; - - size_t ptInd = GetPtInd(); - outPath.AddActiveChartEntry(ptInd, chartEntry); - //cerr << "AFTER AddActiveChartEntry" << endl; - - if (tpsPair.second) { - // there are some rules - //cerr << "symbolbind=" << chartEntry->GetSymbolBind().Debug(mgr.system) << endl; - outPath.AddTargetPhrasesToPath(pool, mgr.system, *this, *tpsPair.second, chartEntry->GetSymbolBind()); - } - } + } else { + // not in cache. Lookup + std::pair<bool, SCFG::TargetPhrases*> tpsPair = CreateTargetPhrasesSCFG(pool, mgr.system, sourcePhrase, key.second); + assert(tpsPair.first && tpsPair.second); + + if (tpsPair.first) { + // new entries + ActiveChartEntryProbing *chartEntry = new (pool.Allocate<ActiveChartEntryProbing>()) ActiveChartEntryProbing(pool, prevEntryCast); + //cerr << "AFTER chartEntry" << endl; + + chartEntry->AddSymbolBindElement(subPhraseRange, wordSought, hypos, *this); + //cerr << "AFTER AddSymbolBindElement" << endl; + + size_t ptInd = GetPtInd(); + outPath.AddActiveChartEntry(ptInd, chartEntry); + //cerr << "AFTER AddActiveChartEntry" << endl; + + if (tpsPair.second) { + // there are some rules + //cerr << "symbolbind=" << chartEntry->GetSymbolBind().Debug(mgr.system) << endl; + outPath.AddTargetPhrasesToPath(pool, mgr.system, *this, *tpsPair.second, chartEntry->GetSymbolBind()); + } + } } } SCFG::TargetPhraseImpl *ProbingPT::CreateTargetPhraseSCFG( - MemPool &pool, - const System &system, - const char *&offset) const + MemPool &pool, + const System &system, + const char *&offset) const { TargetPhraseInfo *tpInfo = (TargetPhraseInfo*) offset; SCFG::TargetPhraseImpl *tp = - new (pool.Allocate<SCFG::TargetPhraseImpl>()) SCFG::TargetPhraseImpl(pool, *this, - system, tpInfo->numWords - 1); + new (pool.Allocate<SCFG::TargetPhraseImpl>()) SCFG::TargetPhraseImpl(pool, *this, + system, tpInfo->numWords - 1); offset += sizeof(TargetPhraseInfo); @@ -633,8 +620,7 @@ SCFG::TargetPhraseImpl *ProbingPT::CreateTargetPhraseSCFG( if (m_engine->num_lex_scores) { tp->scoreProperties = scores + m_engine->num_scores; } - } - else { + } else { // log score 1st SCORE logScores[totalNumScores]; for (size_t i = 0; i < totalNumScores; ++i) { @@ -706,47 +692,47 @@ std::pair<bool, SCFG::TargetPhrases*> ProbingPT::CreateTargetPhrasesSCFG(MemPool /* if (outPath.range.GetStartPos() == 1 || outPath.range.GetStartPos() == 2) { - cerr << "range=" << outPath.range - << " prevEntry=" << prevEntry.GetSymbolBind().Debug(mgr.system) << " " << prevEntryCast.GetKey() - << " wordSought=" << wordSought.Debug(mgr.system) - << " key=" << key.first << " " << key.second - << " query_result=" << query_result.first << " " << (query_result.second == NONE) - << endl; + cerr << "range=" << outPath.range + << " prevEntry=" << prevEntry.GetSymbolBind().Debug(mgr.system) << " " << prevEntryCast.GetKey() + << " wordSought=" << wordSought.Debug(mgr.system) + << " key=" << key.first << " " << key.second + << " query_result=" << query_result.first << " " << (query_result.second == NONE) + << endl; } */ if (query_result.first) { ret.first = true; - size_t ptInd = GetPtInd(); + size_t ptInd = GetPtInd(); - if (query_result.second != NONE) { - // there are some rules - const FeatureFunctions &ffs = system.featureFunctions; + if (query_result.second != NONE) { + // there are some rules + const FeatureFunctions &ffs = system.featureFunctions; - const char *offset = m_engine->memTPS + query_result.second; - uint64_t *numTP = (uint64_t*) offset; - //cerr << "numTP=" << *numTP << endl; + const char *offset = m_engine->memTPS + query_result.second; + uint64_t *numTP = (uint64_t*) offset; + //cerr << "numTP=" << *numTP << endl; - SCFG::TargetPhrases *tps = new (pool.Allocate<SCFG::TargetPhrases>()) SCFG::TargetPhrases(pool, *numTP); - ret.second = tps; + SCFG::TargetPhrases *tps = new (pool.Allocate<SCFG::TargetPhrases>()) SCFG::TargetPhrases(pool, *numTP); + ret.second = tps; - offset += sizeof(uint64_t); - for (size_t i = 0; i < *numTP; ++i) { - SCFG::TargetPhraseImpl *tp = CreateTargetPhraseSCFG(pool, system, offset); - assert(tp); - //cerr << "tp=" << tp->Debug(mgr.system) << endl; + offset += sizeof(uint64_t); + for (size_t i = 0; i < *numTP; ++i) { + SCFG::TargetPhraseImpl *tp = CreateTargetPhraseSCFG(pool, system, offset); + assert(tp); + //cerr << "tp=" << tp->Debug(mgr.system) << endl; - ffs.EvaluateInIsolation(pool, system, sourcePhrase, *tp); + ffs.EvaluateInIsolation(pool, system, sourcePhrase, *tp); - tps->AddTargetPhrase(*tp); + tps->AddTargetPhrase(*tp); - } + } - tps->SortAndPrune(m_tableLimit); - ffs.EvaluateAfterTablePruning(pool, *tps, sourcePhrase); - //cerr << "tps=" << tps->GetSize() << endl; + tps->SortAndPrune(m_tableLimit); + ffs.EvaluateAfterTablePruning(pool, *tps, sourcePhrase); + //cerr << "tps=" << tps->GetSize() << endl; - } + } } return ret; diff --git a/moses2/TranslationModel/ProbingPT/ProbingPT.h b/moses2/TranslationModel/ProbingPT/ProbingPT.h index c5fbefd6f..e0dcf40f1 100644 --- a/moses2/TranslationModel/ProbingPT/ProbingPT.h +++ b/moses2/TranslationModel/ProbingPT/ProbingPT.h @@ -35,35 +35,36 @@ class TargetPhrases; class ProbingPT: public Moses2::PhraseTable { ////////////////////////////////////// - class ActiveChartEntryProbing : public SCFG::ActiveChartEntry - { - typedef SCFG::ActiveChartEntry Parent; - public: + class ActiveChartEntryProbing : public SCFG::ActiveChartEntry + { + typedef SCFG::ActiveChartEntry Parent; + public: - ActiveChartEntryProbing(MemPool &pool) + ActiveChartEntryProbing(MemPool &pool) :Parent(pool) ,m_key(0) - {} + {} - ActiveChartEntryProbing( - MemPool &pool, - const ActiveChartEntryProbing &prevEntry); + ActiveChartEntryProbing( + MemPool &pool, + const ActiveChartEntryProbing &prevEntry); - uint64_t GetKey() const - { return m_key; } + uint64_t GetKey() const { + return m_key; + } - std::pair<bool, uint64_t> GetKey(const SCFG::Word &nextWord, const ProbingPT &pt) const; + std::pair<bool, uint64_t> GetKey(const SCFG::Word &nextWord, const ProbingPT &pt) const; - virtual void AddSymbolBindElement( - const Range &range, - const SCFG::Word &word, - const Moses2::Hypotheses *hypos, - const Moses2::PhraseTable &pt); + virtual void AddSymbolBindElement( + const Range &range, + const SCFG::Word &word, + const Moses2::Hypotheses *hypos, + const Moses2::PhraseTable &pt); - protected: - uint64_t m_key; - }; - ////////////////////////////////////// + protected: + uint64_t m_key; + }; + ////////////////////////////////////// public: ProbingPT(size_t startInd, const std::string &line); @@ -73,20 +74,21 @@ public: virtual void SetParameter(const std::string& key, const std::string& value); void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const; - uint64_t GetUnk() const - { return m_unkId; } + uint64_t GetUnk() const { + return m_unkId; + } // SCFG void InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const; + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; virtual void Lookup(MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const; + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; protected: @@ -101,14 +103,13 @@ protected: void CreateAlignmentMap(System &system, const std::string path); TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const; + InputPath &inputPath) const; TargetPhrases *CreateTargetPhrases(MemPool &pool, const System &system, - const Phrase<Moses2::Word> &sourcePhrase, uint64_t key) const; + const Phrase<Moses2::Word> &sourcePhrase, uint64_t key) const; TargetPhraseImpl *CreateTargetPhrase(MemPool &pool, const System &system, - const char *&offset) const; + const char *&offset) const; - inline const std::pair<bool, const Factor*> *GetTargetFactor(uint32_t probingId) const - { + inline const std::pair<bool, const Factor*> *GetTargetFactor(uint32_t probingId) const { if (probingId >= m_targetVocab.size()) { return NULL; } @@ -118,7 +119,7 @@ protected: std::pair<bool, uint64_t> GetKey(const Phrase<Moses2::Word> &sourcePhrase) const; void GetSourceProbingIds(const Phrase<Moses2::Word> &sourcePhrase, bool &ok, - uint64_t probingSource[]) const; + uint64_t probingSource[]) const; uint64_t GetSourceProbingId(const Word &word) const; @@ -135,22 +136,22 @@ protected: // SCFG void LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; std::pair<bool, SCFG::TargetPhrases*> CreateTargetPhrasesSCFG(MemPool &pool, const System &system, const Phrase<SCFG::Word> &sourcePhrase, uint64_t key) const; // return value: 1st = there are actual rules, not just a empty cell for prefix SCFG::TargetPhraseImpl *CreateTargetPhraseSCFG( - MemPool &pool, - const System &system, - const char *&offset) const; + MemPool &pool, + const System &system, + const char *&offset) const; }; diff --git a/moses2/TranslationModel/ProbingPT/StoreTarget.cpp b/moses2/TranslationModel/ProbingPT/StoreTarget.cpp index 326aaea5f..af0de5c31 100644 --- a/moses2/TranslationModel/ProbingPT/StoreTarget.cpp +++ b/moses2/TranslationModel/ProbingPT/StoreTarget.cpp @@ -17,12 +17,12 @@ namespace Moses2 { StoreTarget::StoreTarget(const std::string &basepath) -:m_basePath(basepath) -,m_vocab(basepath + "/TargetVocab.dat") + :m_basePath(basepath) + ,m_vocab(basepath + "/TargetVocab.dat") { std::string path = basepath + "/TargetColl.dat"; m_fileTargetColl.open(path.c_str(), - std::ios::out | std::ios::binary | std::ios::ate | std::ios::trunc); + std::ios::out | std::ios::binary | std::ios::ate | std::ios::trunc); if (!m_fileTargetColl.is_open()) { throw "can't create file "; } @@ -112,10 +112,10 @@ void StoreTarget::Append(const line_text &line, bool log_prob, bool scfg) vector<bool> nonTerms; util::TokenIter<util::SingleCharacter> it; it = util::TokenIter<util::SingleCharacter>(line.target_phrase, - util::SingleCharacter(' ')); + util::SingleCharacter(' ')); while (it) { - StringPiece word = *it; - //cerr << "word=" << word << endl; + StringPiece word = *it; + //cerr << "word=" << word << endl; bool nonTerm = false; if (scfg) { @@ -129,16 +129,16 @@ void StoreTarget::Append(const line_text &line, bool log_prob, bool scfg) util::TokenIter<util::SingleCharacter> itFactor; itFactor = util::TokenIter<util::SingleCharacter>(word, - util::SingleCharacter('|')); + util::SingleCharacter('|')); while (itFactor) { - StringPiece factor = *itFactor; + StringPiece factor = *itFactor; - string factorStr = factor.as_string(); - uint32_t vocabId = m_vocab.GetVocabId(factorStr); + string factorStr = factor.as_string(); + uint32_t vocabId = m_vocab.GetVocabId(factorStr); - rule->target_phrase.push_back(vocabId); + rule->target_phrase.push_back(vocabId); - itFactor++; + itFactor++; } it++; @@ -146,7 +146,7 @@ void StoreTarget::Append(const line_text &line, bool log_prob, bool scfg) // probs it = util::TokenIter<util::SingleCharacter>(line.prob, - util::SingleCharacter(' ')); + util::SingleCharacter(' ')); while (it) { string tok = it->as_string(); float prob = Scan<float>(tok); @@ -170,7 +170,7 @@ void StoreTarget::Append(const line_text &line, bool log_prob, bool scfg) // alignment it = util::TokenIter<util::SingleCharacter>(line.word_align, - util::SingleCharacter(' ')); + util::SingleCharacter(' ')); while (it) { string tokPair = Trim(it->as_string()); if (tokPair.empty()) { @@ -193,8 +193,7 @@ void StoreTarget::Append(const line_text &line, bool log_prob, bool scfg) rule->word_align_non_term.push_back(sourcePos); rule->word_align_non_term.push_back(targetPos); //cerr << (int) rule->word_all1.back() << " "; - } - else { + } else { rule->word_align_term.push_back(sourcePos); rule->word_align_term.push_back(targetPos); } @@ -221,19 +220,18 @@ void StoreTarget::Append(const line_text &line, bool log_prob, bool scfg) uint32_t StoreTarget::GetAlignId(const std::vector<size_t> &align) { boost::unordered_map<std::vector<size_t>, uint32_t>::iterator iter = - m_aligns.find(align); + m_aligns.find(align); if (iter == m_aligns.end()) { uint32_t ind = m_aligns.size(); m_aligns[align] = ind; return ind; - } - else { + } else { return iter->second; } } void StoreTarget::AppendLexRO(std::string &prop, std::vector<float> &retvector, - bool log_prob) const + bool log_prob) const { size_t startPos = prop.find("{{LexRO "); @@ -258,7 +256,7 @@ void StoreTarget::AppendLexRO(std::string &prop, std::vector<float> &retvector, // exclude LexRO property from property column prop = prop.substr(0, startPos) - + prop.substr(endPos + 2, prop.size() - endPos - 2); + + prop.substr(endPos + 2, prop.size() - endPos - 2); //cerr << "line.property_to_be_binarized=" << line.property_to_be_binarized << "AAAA" << endl; } } diff --git a/moses2/TranslationModel/ProbingPT/StoreTarget.h b/moses2/TranslationModel/ProbingPT/StoreTarget.h index 6fc3b1f66..77ccf08cd 100644 --- a/moses2/TranslationModel/ProbingPT/StoreTarget.h +++ b/moses2/TranslationModel/ProbingPT/StoreTarget.h @@ -43,7 +43,7 @@ protected: void Save(const target_text &rule); void AppendLexRO(std::string &prop, std::vector<float> &retvector, - bool log_prob) const; + bool log_prob) const; }; diff --git a/moses2/TranslationModel/ProbingPT/StoreVocab.h b/moses2/TranslationModel/ProbingPT/StoreVocab.h index e9808707a..3c405af66 100644 --- a/moses2/TranslationModel/ProbingPT/StoreVocab.h +++ b/moses2/TranslationModel/ProbingPT/StoreVocab.h @@ -24,31 +24,27 @@ protected: public: StoreVocab(const std::string &path) - :m_path(path) + :m_path(path) {} virtual ~StoreVocab() {} - VOCABID GetVocabId(const std::string &word) - { + VOCABID GetVocabId(const std::string &word) { typename Coll::iterator iter = m_vocab.find(word); if (iter == m_vocab.end()) { VOCABID ind = m_vocab.size() + 1; m_vocab[word] = ind; return ind; - } - else { + } else { return iter->second; } } - void Insert(VOCABID id, const std::string &word) - { + void Insert(VOCABID id, const std::string &word) { m_vocab[word] = id; } - void Save() - { + void Save() { OutputFileStream strme(m_path); typename Coll::const_iterator iter; diff --git a/moses2/TranslationModel/ProbingPT/hash.cpp b/moses2/TranslationModel/ProbingPT/hash.cpp index aab5ee2b3..a0bda389a 100644 --- a/moses2/TranslationModel/ProbingPT/hash.cpp +++ b/moses2/TranslationModel/ProbingPT/hash.cpp @@ -21,16 +21,16 @@ std::vector<uint64_t> getVocabIDs(const StringPiece &textin) util::TokenIter<util::SingleCharacter> itWord(textin, util::SingleCharacter(' ')); while (itWord) { - StringPiece word = *itWord; - uint64_t id = 0; + StringPiece word = *itWord; + uint64_t id = 0; - util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|')); + util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|')); while (itFactor) { - StringPiece factor = *itFactor; - //cerr << "factor=" << factor << endl; + StringPiece factor = *itFactor; + //cerr << "factor=" << factor << endl; - id += getHash(factor); - itFactor++; + id += getHash(factor); + itFactor++; } output.push_back(id); diff --git a/moses2/TranslationModel/ProbingPT/querying.cpp b/moses2/TranslationModel/ProbingPT/querying.cpp index 9ea2d8cb6..d09bd45a8 100644 --- a/moses2/TranslationModel/ProbingPT/querying.cpp +++ b/moses2/TranslationModel/ProbingPT/querying.cpp @@ -46,10 +46,9 @@ QueryEngine::QueryEngine(const char * filepath, util::LoadMethod load_method) found = Get(keyValue, "API_VERSION", version); if (!found) { std::cerr << "Old or corrupted version of ProbingPT. Please rebinarize your phrase tables." << std::endl; - } - else if (version != API_VERSION) { + } else if (version != API_VERSION) { std::cerr << "The ProbingPT API has changed. " << version << "!=" - << API_VERSION << " Please rebinarize your phrase tables." << std::endl; + << API_VERSION << " Please rebinarize your phrase tables." << std::endl; exit(EXIT_FAILURE); } @@ -143,37 +142,37 @@ void QueryEngine::read_alignments(const std::string &alignPath) void QueryEngine::file_exits(const std::string &basePath) { - if (!FileExists(basePath + "/Alignments.dat")) { - UTIL_THROW2("Require file does not exist in: " << basePath << "/Alignments.dat"); - } - if (!FileExists(basePath + "/TargetColl.dat")) { - UTIL_THROW2("Require file does not exist in: " << basePath << "/TargetColl.dat"); - } - if (!FileExists(basePath + "/TargetVocab.dat")) { - UTIL_THROW2("Require file does not exist in: " << basePath << "/TargetVocab.dat"); - } - if (!FileExists(basePath + "/cache")) { - UTIL_THROW2("Require file does not exist in: " << basePath << "/cache"); - } - if (!FileExists(basePath + "/config")) { - UTIL_THROW2("Require file does not exist in: " << basePath << "/config"); - } - if (!FileExists(basePath + "/probing_hash.dat")) { - UTIL_THROW2("Require file does not exist in: " << basePath << "/probing_hash.dat"); - } - if (!FileExists(basePath + "/source_vocabids")) { - UTIL_THROW2("Require file does not exist in: " << basePath << "/source_vocabids"); - } - - /* - - if (!FileExists(path_to_config) || !FileExists(path_to_hashtable) || - !FileExists(path_to_source_vocabid) || !FileExists(basepath + alignPath) || - !FileExists(basepath + "/TargetColl.dat") || !FileExists(basepath + "/TargetVocab.dat") || - !FileExists(basepath + "/cache")) { - UTIL_THROW2("A required table doesn't exist in: " << basepath); - } - */ + if (!FileExists(basePath + "/Alignments.dat")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/Alignments.dat"); + } + if (!FileExists(basePath + "/TargetColl.dat")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/TargetColl.dat"); + } + if (!FileExists(basePath + "/TargetVocab.dat")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/TargetVocab.dat"); + } + if (!FileExists(basePath + "/cache")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/cache"); + } + if (!FileExists(basePath + "/config")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/config"); + } + if (!FileExists(basePath + "/probing_hash.dat")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/probing_hash.dat"); + } + if (!FileExists(basePath + "/source_vocabids")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/source_vocabids"); + } + + /* + + if (!FileExists(path_to_config) || !FileExists(path_to_hashtable) || + !FileExists(path_to_source_vocabid) || !FileExists(basepath + alignPath) || + !FileExists(basepath + "/TargetColl.dat") || !FileExists(basepath + "/TargetVocab.dat") || + !FileExists(basepath + "/cache")) { + UTIL_THROW2("A required table doesn't exist in: " << basepath); + } + */ } } diff --git a/moses2/TranslationModel/ProbingPT/storing.cpp b/moses2/TranslationModel/ProbingPT/storing.cpp index 75cdcc038..c27e99634 100644 --- a/moses2/TranslationModel/ProbingPT/storing.cpp +++ b/moses2/TranslationModel/ProbingPT/storing.cpp @@ -32,14 +32,12 @@ void Node::Add(Table &table, const SourcePhrase &sourcePhrase, size_t pos) child = &m_children[vocabId]; assert(!child->done); child->key = key + (vocabId << pos); - } - else { + } else { child = &iter->second; } child->Add(table, sourcePhrase, pos + 1); - } - else { + } else { // this node was written previously 'cos it has rules done = true; } @@ -66,8 +64,8 @@ void Node::Write(Table &table) /////////////////////////////////////////////////////////////////////// void createProbingPT(const std::string &phrasetable_path, - const std::string &basepath, int num_scores, int num_lex_scores, - bool log_prob, int max_cache_size, bool scfg) + const std::string &basepath, int num_scores, int num_lex_scores, + bool log_prob, int max_cache_size, bool scfg) { std::cerr << "Starting..." << std::endl; @@ -123,12 +121,10 @@ void createProbingPT(const std::string &phrasetable_path, // 1st line prevSource = line.source_phrase.as_string(); storeTarget.Append(line, log_prob, scfg); - } - else if (prevSource == line.source_phrase) { + } else if (prevSource == line.source_phrase) { //If we still have the same line, just append to it: storeTarget.Append(line, log_prob, scfg); - } - else { + } else { assert(prevSource != line.source_phrase); //Create a new entry even @@ -147,7 +143,7 @@ void createProbingPT(const std::string &phrasetable_path, std::vector<uint64_t> vocabid_source = getVocabIDs(prevSource); if (scfg) { // storing prefixes? - sourcePhrases.Add(sourceEntries, vocabid_source); + sourcePhrases.Add(sourceEntries, vocabid_source); } sourceEntry.key = getKey(vocabid_source); @@ -155,7 +151,7 @@ void createProbingPT(const std::string &phrasetable_path, cerr << "prevSource=" << prevSource << flush << " vocabids=" << Debug(vocabid_source) << flush << " key=" << sourceEntry.key << endl; - */ + */ //Put into table sourceEntries.Insert(sourceEntry); @@ -175,9 +171,9 @@ void createProbingPT(const std::string &phrasetable_path, uint64_t currKey = getKey(currVocabidSource); CacheItem *item = new CacheItem( - Trim(line.source_phrase.as_string()), - currKey, - toks[1]); + Trim(line.source_phrase.as_string()), + currKey, + toks[1]); cache.push(item); if (max_cache_size > 0 && cache.size() > max_cache_size) { @@ -191,8 +187,7 @@ void createProbingPT(const std::string &phrasetable_path, prevSource = line.source_phrase.as_string(); } - } - catch (util::EndOfFileException e) { + } catch (util::EndOfFileException e) { std::cerr << "Reading phrase table finished, writing remaining files to disk." << std::endl; @@ -258,8 +253,8 @@ size_t countUniqueSource(const std::string &path) } void serialize_cache( - std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> &cache, - const std::string &path, float totalSourceCount) + std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> &cache, + const std::string &path, float totalSourceCount) { std::vector<const CacheItem*> vec(cache.size()); diff --git a/moses2/TranslationModel/ProbingPT/vocabid.cpp b/moses2/TranslationModel/ProbingPT/vocabid.cpp index 696373ee5..e752f76bc 100644 --- a/moses2/TranslationModel/ProbingPT/vocabid.cpp +++ b/moses2/TranslationModel/ProbingPT/vocabid.cpp @@ -7,27 +7,27 @@ namespace Moses2 { void add_to_map(StoreVocab<uint64_t> &sourceVocab, - const StringPiece &textin) + const StringPiece &textin) { //Tokenize util::TokenIter<util::SingleCharacter> itWord(textin, util::SingleCharacter(' ')); while (itWord) { - StringPiece word = *itWord; + StringPiece word = *itWord; - util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|')); + util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|')); while (itFactor) { - StringPiece factor = *itFactor; + StringPiece factor = *itFactor; - sourceVocab.Insert(getHash(factor), factor.as_string()); - itFactor++; + sourceVocab.Insert(getHash(factor), factor.as_string()); + itFactor++; } itWord++; } } void serialize_map(const std::map<uint64_t, std::string> &karta, - const std::string &filename) + const std::string &filename) { std::ofstream os(filename.c_str()); diff --git a/moses2/TranslationModel/Transliteration.cpp b/moses2/TranslationModel/Transliteration.cpp index f92348ee9..13c884508 100644 --- a/moses2/TranslationModel/Transliteration.cpp +++ b/moses2/TranslationModel/Transliteration.cpp @@ -28,7 +28,7 @@ namespace Moses2 { Transliteration::Transliteration(size_t startInd, const std::string &line) : - PhraseTable(startInd, line) + PhraseTable(startInd, line) { ReadParameters(); UTIL_THROW_IF2(m_mosesDir.empty() || @@ -63,23 +63,23 @@ SetParameter(const std::string& key, const std::string& value) } void Transliteration::Lookup(const Manager &mgr, - InputPathsBase &inputPaths) const + InputPathsBase &inputPaths) const { - BOOST_FOREACH(InputPathBase *pathBase, inputPaths){ - InputPath *path = static_cast<InputPath*>(pathBase); + BOOST_FOREACH(InputPathBase *pathBase, inputPaths) { + InputPath *path = static_cast<InputPath*>(pathBase); - if (SatisfyBackoff(mgr, *path)) { - const SubPhrase<Moses2::Word> &phrase = path->subPhrase; + if (SatisfyBackoff(mgr, *path)) { + const SubPhrase<Moses2::Word> &phrase = path->subPhrase; - TargetPhrases *tps = Lookup(mgr, mgr.GetPool(), *path); - path->AddTargetPhrases(*this, tps); - } - } + TargetPhrases *tps = Lookup(mgr, mgr.GetPool(), *path); + path->AddTargetPhrases(*this, tps); + } + } } TargetPhrases *Transliteration::Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const + InputPath &inputPath) const { const SubPhrase<Moses2::Word> &sourcePhrase = inputPath.subPhrase; size_t hash = sourcePhrase.hash(); @@ -121,10 +121,10 @@ TargetPhrases *Transliteration::Lookup(const Manager &mgr, MemPool &pool, } std::vector<TargetPhraseImpl*> Transliteration::CreateTargetPhrases( - const Manager &mgr, - MemPool &pool, - const SubPhrase<Moses2::Word> &sourcePhrase, - const std::string &outDir) const + const Manager &mgr, + MemPool &pool, + const SubPhrase<Moses2::Word> &sourcePhrase, + const std::string &outDir) const { std::vector<TargetPhraseImpl*> ret; @@ -137,7 +137,7 @@ std::vector<TargetPhraseImpl*> Transliteration::CreateTargetPhrases( UTIL_THROW_IF2(toks.size() != 2, "Error in transliteration output file. Expecting word\tscore"); TargetPhraseImpl *tp = - new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, *this, mgr.system, 1); + new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, *this, mgr.system, 1); Moses2::Word &word = (*tp)[0]; word.CreateFromString(mgr.system.GetVocab(), mgr.system, toks[0]); @@ -166,61 +166,61 @@ void Transliteration::EvaluateInIsolation(const System &system, // SCFG /////////////////////////////////////////////////////////////////////////////////////////// void Transliteration::InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const { UTIL_THROW2("Not implemented"); } void Transliteration::Lookup(MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const { UTIL_THROW2("Not implemented"); } void Transliteration::LookupUnary(MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const { UTIL_THROW2("Not implemented"); } void Transliteration::LookupNT( - MemPool &pool, - const SCFG::Manager &mgr, - const Moses2::Range &subPhraseRange, - const SCFG::InputPath &prevPath, - const SCFG::Stacks &stacks, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const { UTIL_THROW2("Not implemented"); } void Transliteration::LookupGivenWord( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::InputPath &prevPath, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const { UTIL_THROW2("Not implemented"); } void Transliteration::LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const { UTIL_THROW2("Not implemented"); } diff --git a/moses2/TranslationModel/Transliteration.h b/moses2/TranslationModel/Transliteration.h index 15f262ac8..593677d60 100644 --- a/moses2/TranslationModel/Transliteration.h +++ b/moses2/TranslationModel/Transliteration.h @@ -23,55 +23,55 @@ public: void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const; virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const; + InputPath &inputPath) const; virtual void EvaluateInIsolation(const System &system, const Phrase<Moses2::Word> &source, - const TargetPhraseImpl &targetPhrase, Scores &scores, - SCORE &estimatedScore) const; + const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const; virtual void InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const; + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; void Lookup(MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const; + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; void LookupUnary(MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const; + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; protected: virtual void LookupNT( - MemPool &pool, - const SCFG::Manager &mgr, - const Moses2::Range &subPhraseRange, - const SCFG::InputPath &prevPath, - const SCFG::Stacks &stacks, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const; virtual void LookupGivenWord( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::InputPath &prevPath, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; virtual void LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; void SetParameter(const std::string& key, const std::string& value); @@ -80,10 +80,10 @@ protected: std::string m_mosesDir, m_scriptDir, m_externalDir, m_inputLang, m_outputLang; std::vector<TargetPhraseImpl*> CreateTargetPhrases( - const Manager &mgr, - MemPool &pool, - const SubPhrase<Moses2::Word> &sourcePhrase, - const std::string &outDir) const; + const Manager &mgr, + MemPool &pool, + const SubPhrase<Moses2::Word> &sourcePhrase, + const std::string &outDir) const; }; diff --git a/moses2/TranslationModel/UnknownWordPenalty.cpp b/moses2/TranslationModel/UnknownWordPenalty.cpp index d786b2cff..e165e7e02 100644 --- a/moses2/TranslationModel/UnknownWordPenalty.cpp +++ b/moses2/TranslationModel/UnknownWordPenalty.cpp @@ -26,8 +26,8 @@ namespace Moses2 { UnknownWordPenalty::UnknownWordPenalty(size_t startInd, const std::string &line) -:PhraseTable(startInd, line) -,m_drop(false) + :PhraseTable(startInd, line) + ,m_drop(false) { m_tuneable = false; ReadParameters(); @@ -42,27 +42,24 @@ void UnknownWordPenalty::SetParameter(const std::string& key, const std::string& { if (key == "drop") { m_drop = Scan<bool>(value); - } - else if (key == "prefix") { + } else if (key == "prefix") { m_prefix = value; - } - else if (key == "suffix") { + } else if (key == "suffix") { m_suffix = value; - } - else { + } else { PhraseTable::SetParameter(key, value); } } void UnknownWordPenalty::ProcessXML( - const Manager &mgr, - MemPool &pool, - const Sentence &sentence, - InputPaths &inputPaths) const + const Manager &mgr, + MemPool &pool, + const Sentence &sentence, + InputPaths &inputPaths) const { - const Vector<const InputType::XMLOption*> &xmlOptions = sentence.GetXMLOptions(); - BOOST_FOREACH(const InputType::XMLOption *xmlOption, xmlOptions) { - TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(pool, *this, mgr.system, xmlOption->GetTranslation()); + const Vector<const InputType::XMLOption*> &xmlOptions = sentence.GetXMLOptions(); + BOOST_FOREACH(const InputType::XMLOption *xmlOption, xmlOptions) { + TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(pool, *this, mgr.system, xmlOption->GetTranslation()); if (xmlOption->prob) { Scores &scores = target->GetScores(); @@ -80,22 +77,22 @@ void UnknownWordPenalty::ProcessXML( mgr.system.featureFunctions.EvaluateAfterTablePruning(pool, *tps, source); path->AddTargetPhrases(*this, tps); - } + } } void UnknownWordPenalty::Lookup(const Manager &mgr, - InputPathsBase &inputPaths) const + InputPathsBase &inputPaths) const { - BOOST_FOREACH(InputPathBase *pathBase, inputPaths){ - InputPath *path = static_cast<InputPath*>(pathBase); + BOOST_FOREACH(InputPathBase *pathBase, inputPaths) { + InputPath *path = static_cast<InputPath*>(pathBase); - if (SatisfyBackoff(mgr, *path)) { - const SubPhrase<Moses2::Word> &phrase = path->subPhrase; + if (SatisfyBackoff(mgr, *path)) { + const SubPhrase<Moses2::Word> &phrase = path->subPhrase; - TargetPhrases *tps = Lookup(mgr, mgr.GetPool(), *path); - path->AddTargetPhrases(*this, tps); - } - } + TargetPhrases *tps = Lookup(mgr, mgr.GetPool(), *path); + path->AddTargetPhrases(*this, tps); + } + } } @@ -108,7 +105,7 @@ TargetPhrases *UnknownWordPenalty::Lookup(const Manager &mgr, MemPool &pool, // any other pt translate this? size_t numPt = mgr.system.mappings.size(); const TargetPhrases **allTPS = - static_cast<InputPath&>(inputPath).targetPhrases; + static_cast<InputPath&>(inputPath).targetPhrases; for (size_t i = 0; i < numPt; ++i) { const TargetPhrases *otherTps = allTPS[i]; @@ -126,16 +123,15 @@ TargetPhrases *UnknownWordPenalty::Lookup(const Manager &mgr, MemPool &pool, size_t numWords = m_drop ? 0 : 1; TargetPhraseImpl *target = - new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, *this, - system, numWords); + new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, *this, + system, numWords); if (!m_drop) { Moses2::Word &word = (*target)[0]; if (m_prefix.empty() && m_suffix.empty()) { word[0] = factor; - } - else { + } else { stringstream strm; if (!m_prefix.empty()) { strm << m_prefix; @@ -172,17 +168,17 @@ void UnknownWordPenalty::EvaluateInIsolation(const System &system, // SCFG /////////////////////////////////////////////////////////////////////////////////////////// void UnknownWordPenalty::InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const { } void UnknownWordPenalty::Lookup(MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const { const System &system = mgr.system; @@ -193,7 +189,7 @@ void UnknownWordPenalty::Lookup(MemPool &pool, } if (path.GetNumRules()) { - // only create rules if no other rules + // only create rules if no other rules return; } @@ -240,43 +236,43 @@ void UnknownWordPenalty::Lookup(MemPool &pool, } void UnknownWordPenalty::LookupUnary(MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const { } void UnknownWordPenalty::LookupNT( - MemPool &pool, - const SCFG::Manager &mgr, - const Moses2::Range &subPhraseRange, - const SCFG::InputPath &prevPath, - const SCFG::Stacks &stacks, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const { UTIL_THROW2("Not implemented"); } void UnknownWordPenalty::LookupGivenWord( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::InputPath &prevPath, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const { UTIL_THROW2("Not implemented"); } void UnknownWordPenalty::LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const { UTIL_THROW2("Not implemented"); } diff --git a/moses2/TranslationModel/UnknownWordPenalty.h b/moses2/TranslationModel/UnknownWordPenalty.h index 52c235a36..112f0b6cf 100644 --- a/moses2/TranslationModel/UnknownWordPenalty.h +++ b/moses2/TranslationModel/UnknownWordPenalty.h @@ -25,61 +25,61 @@ public: void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const; virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, - InputPath &inputPath) const; + InputPath &inputPath) const; void ProcessXML( - const Manager &mgr, - MemPool &pool, - const Sentence &sentence, - InputPaths &inputPaths) const; + const Manager &mgr, + MemPool &pool, + const Sentence &sentence, + InputPaths &inputPaths) const; virtual void EvaluateInIsolation(const System &system, const Phrase<Moses2::Word> &source, - const TargetPhraseImpl &targetPhrase, Scores &scores, - SCORE &estimatedScore) const; + const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const; virtual void InitActiveChart( - MemPool &pool, - const SCFG::Manager &mgr, - SCFG::InputPath &path) const; + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; void Lookup(MemPool &pool, - const SCFG::Manager &mgr, - size_t maxChartSpan, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const; + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; void LookupUnary(MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::Stacks &stacks, - SCFG::InputPath &path) const; + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; protected: virtual void LookupNT( - MemPool &pool, - const SCFG::Manager &mgr, - const Moses2::Range &subPhraseRange, - const SCFG::InputPath &prevPath, - const SCFG::Stacks &stacks, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const; virtual void LookupGivenWord( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::InputPath &prevPath, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; virtual void LookupGivenNode( - MemPool &pool, - const SCFG::Manager &mgr, - const SCFG::ActiveChartEntry &prevEntry, - const SCFG::Word &wordSought, - const Moses2::Hypotheses *hypos, - const Moses2::Range &subPhraseRange, - SCFG::InputPath &outPath) const; + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; protected: bool m_drop; std::string m_prefix, m_suffix; |