diff options
author | Hieu Hoang <hieuhoang@gmail.com> | 2017-02-01 01:21:59 +0300 |
---|---|---|
committer | Hieu Hoang <hieuhoang@gmail.com> | 2017-02-01 01:21:59 +0300 |
commit | a8a5b43f2dc32bd1b45006fd43989dc71e74ba0e (patch) | |
tree | e84a78fa005e29ec78076d6e525371240871122c /moses2/TranslationModel | |
parent | 7206d592751ee9afeb1fa4753b7e19272e2585bc (diff) |
move moses2 to root
Diffstat (limited to 'moses2/TranslationModel')
49 files changed, 8710 insertions, 0 deletions
diff --git a/moses2/TranslationModel/CompactPT/BlockHashIndex.cpp b/moses2/TranslationModel/CompactPT/BlockHashIndex.cpp new file mode 100644 index 000000000..338a8e221 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/BlockHashIndex.cpp @@ -0,0 +1,418 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#include "ThrowingFwrite.h" +#include "BlockHashIndex.h" +#include "CmphStringVectorAdapter.h" +#include "util/exception.hh" +#include "util/string_stream.hh" + +#ifdef HAVE_CMPH +#include "cmph.h" +#endif + +namespace Moses2 +{ +#ifdef WITH_THREADS +BlockHashIndex::BlockHashIndex(size_t orderBits, size_t fingerPrintBits, + size_t threadsNum) : + m_orderBits(orderBits), m_fingerPrintBits(fingerPrintBits), m_fileHandle(0), m_fileHandleStart( + 0), m_landmarks(true), m_size(0), m_lastSaved(-1), m_lastDropped(-1), m_numLoadedRanges( + 0), m_threadPool(threadsNum) +{ +#ifndef HAVE_CMPH + std::cerr << "minphr: CMPH support not compiled in." << std::endl; + exit(1); +#endif +} +#else +BlockHashIndex::BlockHashIndex(size_t orderBits, size_t fingerPrintBits) +: m_orderBits(orderBits), m_fingerPrintBits(fingerPrintBits), +m_fileHandle(0), m_fileHandleStart(0), m_size(0), +m_lastSaved(-1), m_lastDropped(-1), m_numLoadedRanges(0) +{ +#ifndef HAVE_CMPH + std::cerr << "minphr: CMPH support not compiled in." << std::endl; + exit(1); +#endif +} +#endif + +BlockHashIndex::~BlockHashIndex() +{ +#ifdef HAVE_CMPH + for (std::vector<void*>::iterator it = m_hashes.begin(); it != m_hashes.end(); + it++) + if (*it != 0) cmph_destroy((cmph_t*) *it); + + for (std::vector<PairedPackedArray<>*>::iterator it = m_arrays.begin(); + it != m_arrays.end(); it++) + if (*it != 0) delete *it; +#endif +} + +size_t BlockHashIndex::GetHash(const char* key) +{ + std::string keyStr(key); + size_t i = std::distance(m_landmarks.begin(), + std::upper_bound(m_landmarks.begin(), m_landmarks.end(), keyStr)) - 1; + + if (i == 0ul - 1) return GetSize(); + + size_t pos = GetHash(i, key); + if (pos != GetSize()) return (1ul << m_orderBits) * i + pos; + else return GetSize(); +} + +size_t BlockHashIndex::GetFprint(const char* key) const +{ + size_t hash; + MurmurHash3_x86_32(key, std::strlen(key), 100000, &hash); + hash &= (1ul << m_fingerPrintBits) - 1; + return hash; +} + +size_t BlockHashIndex::GetHash(size_t i, const char* key) +{ +//#ifdef WITH_THREADS +// boost::mutex::scoped_lock lock(m_mutex); +//#endif + //if(m_hashes[i] == 0) + //LoadRange(i); +#ifdef HAVE_CMPH + size_t idx = cmph_search((cmph_t*) m_hashes[i], key, + (cmph_uint32) strlen(key)); +#else + assert(0); + size_t idx = 0; +#endif + + std::pair<size_t, size_t> orderPrint = m_arrays[i]->Get(idx, m_orderBits, + m_fingerPrintBits); + m_clocks[i] = clock(); + + if (GetFprint(key) == orderPrint.second) return orderPrint.first; + else return GetSize(); +} + +size_t BlockHashIndex::GetHash(std::string key) +{ + return GetHash(key.c_str()); +} + +size_t BlockHashIndex::operator[](std::string key) +{ + return GetHash(key); +} + +size_t BlockHashIndex::operator[](char* key) +{ + return GetHash(key); +} + +size_t BlockHashIndex::Save(std::string filename) +{ + std::FILE* mphf = std::fopen(filename.c_str(), "w"); + size_t size = Save(mphf); + std::fclose(mphf); + return size; +} + +void BlockHashIndex::BeginSave(std::FILE * mphf) +{ + m_fileHandle = mphf; + ThrowingFwrite(&m_orderBits, sizeof(size_t), 1, m_fileHandle); + ThrowingFwrite(&m_fingerPrintBits, sizeof(size_t), 1, m_fileHandle); + + m_fileHandleStart = std::ftell(m_fileHandle); + + size_t relIndexPos = 0; + ThrowingFwrite(&relIndexPos, sizeof(size_t), 1, m_fileHandle); +} + +void BlockHashIndex::SaveRange(size_t i) +{ +#ifdef HAVE_CMPH + if (m_seekIndex.size() <= i) m_seekIndex.resize(i + 1); + m_seekIndex[i] = std::ftell(m_fileHandle) - m_fileHandleStart; + cmph_dump((cmph_t*) m_hashes[i], m_fileHandle); + m_arrays[i]->Save(m_fileHandle); +#endif +} + +void BlockHashIndex::SaveLastRange() +{ +#ifdef WITH_THREADS + boost::mutex::scoped_lock lock(m_mutex); +#endif + + while (!m_queue.empty() && m_lastSaved + 1 == -m_queue.top()) { + size_t current = -m_queue.top(); + m_queue.pop(); + SaveRange(current); + m_lastSaved = current; + } +} + +void BlockHashIndex::DropRange(size_t i) +{ +#ifdef HAVE_CMPH + if (m_hashes[i] != 0) { + cmph_destroy((cmph_t*) m_hashes[i]); + m_hashes[i] = 0; + } + if (m_arrays[i] != 0) { + delete m_arrays[i]; + m_arrays[i] = 0; + m_clocks[i] = 0; + } + m_numLoadedRanges--; +#endif +} + +void BlockHashIndex::DropLastRange() +{ +#ifdef WITH_THREADS + boost::mutex::scoped_lock lock(m_mutex); +#endif + + while (m_lastDropped != m_lastSaved) + DropRange(++m_lastDropped); +} + +#ifdef WITH_THREADS +void BlockHashIndex::WaitAll() +{ + m_threadPool.Stop(true); +} +#endif + +size_t BlockHashIndex::FinalizeSave() +{ +#ifdef WITH_THREADS + m_threadPool.Stop(true); +#endif + + SaveLastRange(); + + size_t relIndexPos = std::ftell(m_fileHandle) - m_fileHandleStart; + + std::fseek(m_fileHandle, m_fileHandleStart, SEEK_SET); + ThrowingFwrite(&relIndexPos, sizeof(size_t), 1, m_fileHandle); + + std::fseek(m_fileHandle, m_fileHandleStart + relIndexPos, SEEK_SET); + m_landmarks.save(m_fileHandle); + + size_t seekIndexSize = m_seekIndex.size(); + ThrowingFwrite(&seekIndexSize, sizeof(size_t), 1, m_fileHandle); + ThrowingFwrite(&m_seekIndex[0], sizeof(size_t), seekIndexSize, m_fileHandle); + + ThrowingFwrite(&m_size, sizeof(size_t), 1, m_fileHandle); + + size_t fileHandleStop = std::ftell(m_fileHandle); + return fileHandleStop - m_fileHandleStart + sizeof(m_orderBits) + + sizeof(m_fingerPrintBits); +} + +size_t BlockHashIndex::Save(std::FILE * mphf) +{ + m_queue = std::priority_queue<int>(); + BeginSave(mphf); + for (size_t i = 0; i < m_hashes.size(); i++) + SaveRange(i); + return FinalizeSave(); +} + +size_t BlockHashIndex::LoadIndex(std::FILE* mphf) +{ + m_fileHandle = mphf; + + size_t beginning = std::ftell(mphf); + + size_t read = 0; + read += std::fread(&m_orderBits, sizeof(size_t), 1, mphf); + read += std::fread(&m_fingerPrintBits, sizeof(size_t), 1, mphf); + m_fileHandleStart = std::ftell(m_fileHandle); + + size_t relIndexPos; + read += std::fread(&relIndexPos, sizeof(size_t), 1, mphf); + std::fseek(m_fileHandle, m_fileHandleStart + relIndexPos, SEEK_SET); + + m_landmarks.load(mphf); + + size_t seekIndexSize; + read += std::fread(&seekIndexSize, sizeof(size_t), 1, m_fileHandle); + m_seekIndex.resize(seekIndexSize); + read += std::fread(&m_seekIndex[0], sizeof(size_t), seekIndexSize, + m_fileHandle); + m_hashes.resize(seekIndexSize, 0); + m_clocks.resize(seekIndexSize, 0); + m_arrays.resize(seekIndexSize, 0); + + read += std::fread(&m_size, sizeof(size_t), 1, m_fileHandle); + + size_t end = std::ftell(mphf); + + return end - beginning; +} + +void BlockHashIndex::LoadRange(size_t i) +{ +#ifdef HAVE_CMPH + std::fseek(m_fileHandle, m_fileHandleStart + m_seekIndex[i], SEEK_SET); + cmph_t* hash = cmph_load(m_fileHandle); + m_arrays[i] = new PairedPackedArray<>(0, m_orderBits, m_fingerPrintBits); + m_arrays[i]->Load(m_fileHandle); + + m_hashes[i] = (void*) hash; + m_clocks[i] = clock(); + + m_numLoadedRanges++; +#endif +} + +size_t BlockHashIndex::Load(std::string filename) +{ + std::FILE* mphf = std::fopen(filename.c_str(), "r"); + size_t size = Load(mphf); + std::fclose(mphf); + return size; +} + +size_t BlockHashIndex::Load(std::FILE * mphf) +{ + size_t byteSize = LoadIndex(mphf); + size_t end = std::ftell(mphf); + + for (size_t i = 0; i < m_seekIndex.size(); i++) + LoadRange(i); + std::fseek(m_fileHandle, end, SEEK_SET); + return byteSize; +} + +size_t BlockHashIndex::GetSize() const +{ + return m_size; +} + +void BlockHashIndex::KeepNLastRanges(float ratio, float tolerance) +{ + /* + #ifdef WITH_THREADS + boost::mutex::scoped_lock lock(m_mutex); + #endif + size_t n = m_hashes.size() * ratio; + size_t max = n * (1 + tolerance); + if(m_numLoadedRanges > max) { + typedef std::vector<std::pair<clock_t, size_t> > LastLoaded; + LastLoaded lastLoaded; + for(size_t i = 0; i < m_hashes.size(); i++) + if(m_hashes[i] != 0) + lastLoaded.push_back(std::make_pair(m_clocks[i], i)); + + std::sort(lastLoaded.begin(), lastLoaded.end()); + for(LastLoaded::reverse_iterator it = lastLoaded.rbegin() + size_t(n * (1 - tolerance)); + it != lastLoaded.rend(); it++) + DropRange(it->second); + }*/ +} + +void BlockHashIndex::CalcHash(size_t current, void* source_void) +{ +#ifdef HAVE_CMPH + cmph_io_adapter_t* source = (cmph_io_adapter_t*) source_void; + cmph_config_t *config = cmph_config_new(source); + cmph_config_set_algo(config, CMPH_CHD); + + cmph_t* hash = cmph_new(config); + PairedPackedArray<> *pv = new PairedPackedArray<>(source->nkeys, m_orderBits, + m_fingerPrintBits); + + size_t i = 0; + + source->rewind(source->data); + + std::string lastKey = ""; + while (i < source->nkeys) { + unsigned keylen; + char* key; + source->read(source->data, &key, &keylen); + std::string temp(key, keylen); + source->dispose(source->data, key, keylen); + + if (lastKey > temp) { + if (source->nkeys != 2 || temp != "###DUMMY_KEY###") { + util::StringStream strme; + strme + << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n"; + strme << "1: " << lastKey << "\n"; + strme << "2: " << temp << "\n"; + UTIL_THROW2(strme.str()); + } + } + lastKey = temp; + + size_t fprint = GetFprint(temp.c_str()); + size_t idx = cmph_search(hash, temp.c_str(), (cmph_uint32) temp.size()); + + pv->Set(idx, i, fprint, m_orderBits, m_fingerPrintBits); + i++; + } + + cmph_config_destroy(config); + +#ifdef WITH_THREADS + boost::mutex::scoped_lock lock(m_mutex); +#endif + + if (m_hashes.size() <= current) { + m_hashes.resize(current + 1, 0); + m_arrays.resize(current + 1, 0); + m_clocks.resize(current + 1, 0); + } + + m_hashes[current] = (void*) hash; + m_arrays[current] = pv; + m_clocks[current] = clock(); + m_queue.push(-current); +#endif +} + +#ifdef HAVE_CMPH +void* BlockHashIndex::vectorAdapter(std::vector<std::string>& v) +{ + return (void*) CmphVectorAdapter(v); +} + +void* BlockHashIndex::vectorAdapter( + StringVector<unsigned, size_t, std::allocator>& sv) +{ + return (void*) CmphStringVectorAdapter(sv); +} + +void* BlockHashIndex::vectorAdapter( + StringVector<unsigned, size_t, MmapAllocator>& sv) +{ + return (void*) CmphStringVectorAdapter(sv); +} +#endif + +} diff --git a/moses2/TranslationModel/CompactPT/BlockHashIndex.h b/moses2/TranslationModel/CompactPT/BlockHashIndex.h new file mode 100644 index 000000000..b91ef8f6c --- /dev/null +++ b/moses2/TranslationModel/CompactPT/BlockHashIndex.h @@ -0,0 +1,200 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_BlockHashIndex_h +#define moses_BlockHashIndex_h + +#include <iostream> +#include <sstream> +#include <string> +#include <vector> +#include <queue> +#include <cstring> +#include <cstdio> +#include <boost/thread.hpp> + +#include "MurmurHash3.h" +#include "StringVector.h" +#include "PackedArray.h" +#include "util/exception.hh" +#include "util/string_stream.hh" + +#ifdef WITH_THREADS +#include "../../legacy/ThreadPool.h" +#else +#include <ctime> +#endif + +#include <boost/shared_ptr.hpp> + +namespace Moses2 +{ + +class BlockHashIndex +{ +private: + std::priority_queue<int> m_queue; + + size_t m_orderBits; + size_t m_fingerPrintBits; + + std::FILE* m_fileHandle; + size_t m_fileHandleStart; + + StringVector<unsigned char, unsigned long> m_landmarks; + + std::vector<void*> m_hashes; + std::vector<clock_t> m_clocks; + std::vector<PairedPackedArray<>*> m_arrays; + + std::vector<size_t> m_seekIndex; + + size_t m_size; + int m_lastSaved; + int m_lastDropped; + size_t m_numLoadedRanges; + +#ifdef WITH_THREADS + ThreadPool m_threadPool; + boost::mutex m_mutex; + + template<typename Keys> + class HashTask: public Task + { + public: + HashTask(int id, BlockHashIndex& hash, Keys& keys) : + m_id(id), m_hash(hash), m_keys(new Keys(keys)) + { + } + + virtual void Run() + { + m_hash.CalcHash(m_id, *m_keys); + } + + virtual ~HashTask() + { + delete m_keys; + } + + private: + int m_id; + BlockHashIndex& m_hash; + Keys* m_keys; + }; +#endif + + size_t GetFprint(const char* key) const; + size_t GetHash(size_t i, const char* key); + +public: +#ifdef WITH_THREADS + BlockHashIndex(size_t orderBits, size_t fingerPrintBits, + size_t threadsNum = 2); +#else + BlockHashIndex(size_t orderBits, size_t fingerPrintBits); +#endif + + ~BlockHashIndex(); + + size_t GetHash(const char* key); + size_t GetHash(std::string key); + + size_t operator[](std::string key); + size_t operator[](char* key); + + void BeginSave(std::FILE* mphf); + void SaveRange(size_t i); + void SaveLastRange(); + size_t FinalizeSave(); + +#ifdef WITH_THREADS + void WaitAll(); +#endif + + void DropRange(size_t i); + void DropLastRange(); + + size_t LoadIndex(std::FILE* mphf); + void LoadRange(size_t i); + + size_t Save(std::string filename); + size_t Save(std::FILE * mphf); + + size_t Load(std::string filename); + size_t Load(std::FILE * mphf); + + size_t GetSize() const; + + void KeepNLastRanges(float ratio = 0.1, float tolerance = 0.1); + + template<typename Keys> + void AddRange(Keys &keys) + { + size_t current = m_landmarks.size(); + + if (m_landmarks.size() && m_landmarks.back().str() >= keys[0]) { + util::StringStream strme; + strme + << "ERROR: Input file does not appear to be sorted with LC_ALL=C sort\n"; + strme << "1: " << m_landmarks.back().str() << "\n"; + strme << "2: " << keys[0] << "\n"; + UTIL_THROW2(strme.str()); + } + + m_landmarks.push_back(keys[0]); + m_size += keys.size(); + + if (keys.size() == 1) { + // add dummy key to avoid null hash + keys.push_back("###DUMMY_KEY###"); + } + +#ifdef WITH_THREADS + + boost::shared_ptr<HashTask<Keys> > ht( + new HashTask<Keys>(current, *this, keys)); + m_threadPool.Submit(ht); +#else + CalcHash(current, keys); +#endif + } + + template<typename Keys> + void CalcHash(size_t current, Keys &keys) + { +#ifdef HAVE_CMPH + void* source = vectorAdapter(keys); + CalcHash(current, source); +#endif + } + + void CalcHash(size_t current, void* source); + +#ifdef HAVE_CMPH + void* vectorAdapter(std::vector<std::string>& v); + void* vectorAdapter(StringVector<unsigned, size_t, std::allocator>& sv); + void* vectorAdapter(StringVector<unsigned, size_t, MmapAllocator>& sv); +#endif +}; + +} +#endif diff --git a/moses2/TranslationModel/CompactPT/CanonicalHuffman.h b/moses2/TranslationModel/CompactPT/CanonicalHuffman.h new file mode 100644 index 000000000..ffb6488c0 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/CanonicalHuffman.h @@ -0,0 +1,345 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_CanonicalHuffman_h +#define moses_CanonicalHuffman_h + +#include <string> +#include <algorithm> +#include <boost/dynamic_bitset.hpp> +#include <boost/unordered_map.hpp> + +#include "ThrowingFwrite.h" + +namespace Moses2 +{ + +template<typename Data> +class CanonicalHuffman +{ +private: + std::vector<Data> m_symbols; + std::vector<size_t> m_firstCodes; + std::vector<size_t> m_lengthIndex; + + typedef boost::unordered_map<Data, boost::dynamic_bitset<> > EncodeMap; + EncodeMap m_encodeMap; + + struct MinHeapSorter + { + std::vector<size_t>& m_vec; + + MinHeapSorter(std::vector<size_t>& vec) : + m_vec(vec) + { + } + + bool operator()(size_t a, size_t b) + { + return m_vec[a] > m_vec[b]; + } + }; + + template<class Iterator> + void CalcLengths(Iterator begin, Iterator end, std::vector<size_t>& lengths) + { + size_t n = std::distance(begin, end); + std::vector<size_t> A(2 * n, 0); + + m_symbols.resize(n); + size_t i = 0; + for (Iterator it = begin; it != end; it++) { + m_symbols[i] = it->first; + + A[i] = n + i; + A[n + i] = it->second; + i++; + } + + if (n == 1) { + lengths.push_back(1); + return; + } + + MinHeapSorter hs(A); + std::make_heap(A.begin(), A.begin() + n, hs); + + size_t h = n; + size_t m1, m2; + while (h > 1) { + m1 = A[0]; + std::pop_heap(A.begin(), A.begin() + h, hs); + + h--; + + m2 = A[0]; + std::pop_heap(A.begin(), A.begin() + h, hs); + + A[h] = A[m1] + A[m2]; + A[h - 1] = h; + A[m1] = A[m2] = h; + + std::push_heap(A.begin(), A.begin() + h, hs); + } + + A[1] = 0; + for (size_t i = 2; i < 2 * n; i++) + A[i] = A[A[i]] + 1; + + lengths.resize(n); + for (size_t i = 0; i < n; i++) + lengths[i] = A[i + n]; + } + + void CalcCodes(std::vector<size_t>& lengths) + { + std::vector<size_t> numLength; + for (std::vector<size_t>::iterator it = lengths.begin(); + it != lengths.end(); it++) { + size_t length = *it; + if (numLength.size() <= length) numLength.resize(length + 1, 0); + numLength[length]++; + } + + m_lengthIndex.resize(numLength.size()); + m_lengthIndex[0] = 0; + for (size_t l = 1; l < numLength.size(); l++) + m_lengthIndex[l] = m_lengthIndex[l - 1] + numLength[l - 1]; + + size_t maxLength = numLength.size() - 1; + + m_firstCodes.resize(maxLength + 1, 0); + for (size_t l = maxLength - 1; l > 0; l--) + m_firstCodes[l] = (m_firstCodes[l + 1] + numLength[l + 1]) / 2; + + std::vector<Data> t_symbols; + t_symbols.resize(lengths.size()); + + std::vector<size_t> nextCode = m_firstCodes; + for (size_t i = 0; i < lengths.size(); i++) { + Data data = m_symbols[i]; + size_t length = lengths[i]; + + size_t pos = m_lengthIndex[length] + + (nextCode[length] - m_firstCodes[length]); + t_symbols[pos] = data; + + nextCode[length] = nextCode[length] + 1; + } + + m_symbols.swap(t_symbols); + } + + void CreateCodeMap() + { + for (size_t l = 1; l < m_lengthIndex.size(); l++) { + size_t intCode = m_firstCodes[l]; + size_t num = ( + (l + 1 < m_lengthIndex.size()) ? + m_lengthIndex[l + 1] : m_symbols.size()) - m_lengthIndex[l]; + + for (size_t i = 0; i < num; i++) { + Data data = m_symbols[m_lengthIndex[l] + i]; + boost::dynamic_bitset<> bitCode(l, intCode); + m_encodeMap[data] = bitCode; + intCode++; + } + } + } + + const boost::dynamic_bitset<>& Encode(Data data) const + { + typename EncodeMap::const_iterator it = m_encodeMap.find(data); + UTIL_THROW_IF2(it == m_encodeMap.end(), + "Cannot find symbol in encoding map"); + return it->second; + } + + template<class BitWrapper> + void PutCode(BitWrapper& bitWrapper, const boost::dynamic_bitset<>& code) + { + for (int j = code.size() - 1; j >= 0; j--) + bitWrapper.Put(code[j]); + } + +public: + + template<class Iterator> + CanonicalHuffman(Iterator begin, Iterator end, bool forEncoding = true) + { + std::vector<size_t> lengths; + CalcLengths(begin, end, lengths); + CalcCodes(lengths); + + if (forEncoding) CreateCodeMap(); + } + + CanonicalHuffman(std::FILE* pFile, bool forEncoding = false) + { + Load(pFile); + + if (forEncoding) CreateCodeMap(); + } + + template<class BitWrapper> + void Put(BitWrapper& bitWrapper, Data data) + { + PutCode(bitWrapper, Encode(data)); + } + + template<class BitWrapper> + Data Read(BitWrapper& bitWrapper) + { + if (bitWrapper.TellFromEnd()) { + size_t intCode = bitWrapper.Read(); + size_t len = 1; + while (intCode < m_firstCodes[len]) { + intCode = 2 * intCode + bitWrapper.Read(); + len++; + } + return m_symbols[m_lengthIndex[len] + (intCode - m_firstCodes[len])]; + } + return Data(); + } + + size_t Load(std::FILE* pFile) + { + size_t start = std::ftell(pFile); + size_t read = 0; + + size_t size; + read += std::fread(&size, sizeof(size_t), 1, pFile); + m_symbols.resize(size); + read += std::fread(&m_symbols[0], sizeof(Data), size, pFile); + + read += std::fread(&size, sizeof(size_t), 1, pFile); + m_firstCodes.resize(size); + read += std::fread(&m_firstCodes[0], sizeof(size_t), size, pFile); + + read += std::fread(&size, sizeof(size_t), 1, pFile); + m_lengthIndex.resize(size); + read += std::fread(&m_lengthIndex[0], sizeof(size_t), size, pFile); + + return std::ftell(pFile) - start; + } + + size_t Save(std::FILE* pFile) + { + size_t start = std::ftell(pFile); + + size_t size = m_symbols.size(); + ThrowingFwrite(&size, sizeof(size_t), 1, pFile); + ThrowingFwrite(&m_symbols[0], sizeof(Data), size, pFile); + + size = m_firstCodes.size(); + ThrowingFwrite(&size, sizeof(size_t), 1, pFile); + ThrowingFwrite(&m_firstCodes[0], sizeof(size_t), size, pFile); + + size = m_lengthIndex.size(); + ThrowingFwrite(&size, sizeof(size_t), 1, pFile); + ThrowingFwrite(&m_lengthIndex[0], sizeof(size_t), size, pFile); + + return std::ftell(pFile) - start; + } +}; + +template<class Container = std::string> +class BitWrapper +{ +private: + Container& m_data; + + typename Container::iterator m_iterator; + typename Container::value_type m_currentValue; + + size_t m_valueBits; + typename Container::value_type m_mask; + size_t m_bitPos; + +public: + + BitWrapper(Container &data) : + m_data(data), m_iterator(m_data.begin()), m_currentValue(0), m_valueBits( + sizeof(typename Container::value_type) * 8), m_mask(1), m_bitPos(0) + { + } + + bool Read() + { + if (m_bitPos % m_valueBits == 0) { + if (m_iterator != m_data.end()) m_currentValue = *m_iterator++; + } + else m_currentValue = m_currentValue >> 1; + + m_bitPos++; + return (m_currentValue & m_mask); + } + + void Put(bool bit) + { + if (m_bitPos % m_valueBits == 0) m_data.push_back(0); + + if (bit) m_data[m_data.size() - 1] |= m_mask << (m_bitPos % m_valueBits); + + m_bitPos++; + } + + size_t Tell() + { + return m_bitPos; + } + + size_t TellFromEnd() + { + if (m_data.size() * m_valueBits < m_bitPos) return 0; + return m_data.size() * m_valueBits - m_bitPos; + } + + void Seek(size_t bitPos) + { + m_bitPos = bitPos; + m_iterator = m_data.begin() + int((m_bitPos - 1) / m_valueBits); + m_currentValue = (*m_iterator) >> ((m_bitPos - 1) % m_valueBits); + m_iterator++; + } + + void SeekFromEnd(size_t bitPosFromEnd) + { + size_t bitPos = m_data.size() * m_valueBits - bitPosFromEnd; + Seek(bitPos); + } + + void Reset() + { + m_iterator = m_data.begin(); + m_currentValue = 0; + m_bitPos = 0; + } + + Container& GetContainer() + { + return m_data; + } +}; + +} + +#endif diff --git a/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.cpp b/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.cpp new file mode 100644 index 000000000..a51dc5a45 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.cpp @@ -0,0 +1,95 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifdef HAVE_CMPH + +#include "CmphStringVectorAdapter.h" + +namespace Moses2 +{ + +void CmphStringVectorAdapterDispose(void *data, char *key, cmph_uint32 keylen) +{ + delete[] key; +} + +void CmphStringVectorAdapterRewind(void *data) +{ + cmph_vector_t *cmph_vector = (cmph_vector_t *) data; + cmph_vector->position = 0; +} + +//************************************************************************// + +cmph_io_adapter_t *CmphVectorAdapterNew(std::vector<std::string>& v) +{ + cmph_io_adapter_t * key_source = (cmph_io_adapter_t *) malloc( + sizeof(cmph_io_adapter_t)); + cmph_vector_t * cmph_vector = (cmph_vector_t *) malloc(sizeof(cmph_vector_t)); + assert(key_source); + assert(cmph_vector); + + cmph_vector->vector = (void *) &v; + cmph_vector->position = 0; + key_source->data = (void *) cmph_vector; + key_source->nkeys = v.size(); + + return key_source; +} + +int CmphVectorAdapterRead(void *data, char **key, cmph_uint32 *keylen) +{ + cmph_vector_t *cmph_vector = (cmph_vector_t *) data; + std::vector<std::string>* v = (std::vector<std::string>*) cmph_vector->vector; + size_t size; + *keylen = (*v)[cmph_vector->position].size(); + size = *keylen; + *key = new char[size + 1]; + std::string temp = (*v)[cmph_vector->position]; + strcpy(*key, temp.c_str()); + cmph_vector->position = cmph_vector->position + 1; + return (int) (*keylen); +} + +void CmphVectorAdapterDispose(void *data, char *key, cmph_uint32 keylen) +{ + delete[] key; +} + +void CmphVectorAdapterRewind(void *data) +{ + cmph_vector_t *cmph_vector = (cmph_vector_t *) data; + cmph_vector->position = 0; +} + +cmph_io_adapter_t* CmphVectorAdapter(std::vector<std::string>& v) +{ + cmph_io_adapter_t * key_source = CmphVectorAdapterNew(v); + + key_source->read = CmphVectorAdapterRead; + key_source->dispose = CmphVectorAdapterDispose; + key_source->rewind = CmphVectorAdapterRewind; + return key_source; +} + +} + +#endif diff --git a/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.h b/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.h new file mode 100644 index 000000000..20d43a80c --- /dev/null +++ b/moses2/TranslationModel/CompactPT/CmphStringVectorAdapter.h @@ -0,0 +1,108 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_CmphStringVectorAdapterNew_h +#define moses_CmphStringVectorAdapterNew_h + +#include <cassert> +#include <cstring> + +#ifdef HAVE_CMPH +#include "cmph.h" + +#include "StringVector.h" + +namespace Moses2 +{ + +typedef struct +{ + void *vector; + cmph_uint32 position; +} cmph_vector_t; + +template<typename ValueT, typename PosT, template<typename > class Allocator> +cmph_io_adapter_t *CmphStringVectorAdapterNew( + StringVector<ValueT, PosT, Allocator>& sv) +{ + cmph_io_adapter_t * key_source = (cmph_io_adapter_t *) malloc( + sizeof(cmph_io_adapter_t)); + cmph_vector_t * cmph_vector = (cmph_vector_t *) malloc(sizeof(cmph_vector_t)); + assert(key_source); + assert(cmph_vector); + + cmph_vector->vector = (void *) &sv; + cmph_vector->position = 0; + key_source->data = (void *) cmph_vector; + key_source->nkeys = sv.size(); + + return key_source; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +int CmphStringVectorAdapterRead(void *data, char **key, cmph_uint32 *keylen) +{ + cmph_vector_t *cmph_vector = (cmph_vector_t *) data; + StringVector<ValueT, PosT, Allocator>* sv = (StringVector<ValueT, PosT, + Allocator>*) cmph_vector->vector; + size_t size; + *keylen = (*sv)[cmph_vector->position].size(); + size = *keylen; + *key = new char[size + 1]; + std::string temp = (*sv)[cmph_vector->position]; + std::strcpy(*key, temp.c_str()); + cmph_vector->position = cmph_vector->position + 1; + return (int) (*keylen); +} + +void CmphStringVectorAdapterDispose(void *data, char *key, cmph_uint32 keylen); + +void CmphStringVectorAdapterRewind(void *data); + +template<typename ValueT, typename PosT, template<typename > class Allocator> +cmph_io_adapter_t* CmphStringVectorAdapter( + StringVector<ValueT, PosT, Allocator>& sv) +{ + cmph_io_adapter_t * key_source = CmphStringVectorAdapterNew(sv); + + key_source->read = CmphStringVectorAdapterRead<ValueT, PosT, Allocator>; + key_source->dispose = CmphStringVectorAdapterDispose; + key_source->rewind = CmphStringVectorAdapterRewind; + return key_source; +} + +//************************************************************************// + +cmph_io_adapter_t *CmphVectorAdapterNew(std::vector<std::string>& v); + +int CmphVectorAdapterRead(void *data, char **key, cmph_uint32 *keylen); + +void CmphVectorAdapterDispose(void *data, char *key, cmph_uint32 keylen); + +void CmphVectorAdapterRewind(void *data); + +cmph_io_adapter_t* CmphVectorAdapter(std::vector<std::string>& v); + +} + +#endif + +#endif diff --git a/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp b/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp new file mode 100644 index 000000000..1d32b9a6f --- /dev/null +++ b/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.cpp @@ -0,0 +1,173 @@ +// -*- c++ -*- +// vim:tabstop=2 +// $Id$ +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#include "LexicalReorderingTableCompact.h" +#include "../../SubPhrase.h" +#include "../../legacy/Util2.h" + +namespace Moses2 +{ + +////////////////////////////////////////////////////////////////////////////////////////////// + +bool LexicalReorderingTableCompact::s_inMemoryByDefault = false; + +LexicalReorderingTableCompact::LexicalReorderingTableCompact( + const std::string& filePath, const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) : + LexicalReorderingTable(f_factors, e_factors, c_factors), m_inMemory( + s_inMemoryByDefault), m_numScoreComponent(6), m_multipleScoreTrees( + true), m_hash(10, 16), m_scoreTrees(1) +{ + Load(filePath); +} + +LexicalReorderingTableCompact::LexicalReorderingTableCompact( + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) : + LexicalReorderingTable(f_factors, e_factors, c_factors), m_inMemory( + s_inMemoryByDefault), m_numScoreComponent(6), m_multipleScoreTrees( + true), m_hash(10, 16), m_scoreTrees(1) +{ +} + +LexicalReorderingTableCompact::~LexicalReorderingTableCompact() +{ + for (size_t i = 0; i < m_scoreTrees.size(); i++) + delete m_scoreTrees[i]; +} + +std::vector<float> LexicalReorderingTableCompact::GetScore(const Phrase<Moses2::Word>& f, + const Phrase<Moses2::Word>& e, const Phrase<Moses2::Word>& c) +{ + std::string key; + std::vector<float> scores; + + if (0 == c.GetSize()) key = MakeKey(f, e, c); + else { + for (size_t i = 0; i <= c.GetSize(); ++i) { + SubPhrase<Moses2::Word> sub_c = c.GetSubPhrase(i, c.GetSize() - i); + key = MakeKey(f, e, sub_c); + } + } + + size_t index = m_hash[key]; + if (m_hash.GetSize() != index) { + std::string scoresString; + if (m_inMemory) scoresString = m_scoresMemory[index].str(); + else scoresString = m_scoresMapped[index].str(); + + BitWrapper<> bitStream(scoresString); + for (size_t i = 0; i < m_numScoreComponent; i++) + scores.push_back( + m_scoreTrees[m_multipleScoreTrees ? i : 0]->Read(bitStream)); + + return scores; + } + + return std::vector<float>(); +} + +std::string LexicalReorderingTableCompact::MakeKey(const Phrase<Moses2::Word>& f, + const Phrase<Moses2::Word>& e, const Phrase<Moses2::Word>& c) const +{ + return MakeKey(Trim(f.GetString(m_FactorsF)), Trim(e.GetString(m_FactorsE)), + Trim(c.GetString(m_FactorsC))); +} + +std::string LexicalReorderingTableCompact::MakeKey(const std::string& f, + const std::string& e, const std::string& c) const +{ + std::string key; + if (!f.empty()) key += f; + if (!m_FactorsE.empty()) { + if (!key.empty()) key += " ||| "; + key += e; + } + if (!m_FactorsC.empty()) { + if (!key.empty()) key += " ||| "; + key += c; + } + key += " ||| "; + return key; +} + +LexicalReorderingTable* +LexicalReorderingTableCompact::CheckAndLoad(const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors) +{ +#ifdef HAVE_CMPH + std::string minlexr = ".minlexr"; + // file name is specified without suffix + if (FileExists(filePath + minlexr)) { + //there exists a compact binary version use that + std::cerr << "Using compact lexical reordering table" << std::endl; + return new LexicalReorderingTableCompact(filePath + minlexr, f_factors, + e_factors, c_factors); + } + // file name is specified with suffix + if (filePath.substr(filePath.length() - minlexr.length(), minlexr.length()) + == minlexr && FileExists(filePath)) { + //there exists a compact binary version use that + std::cerr << "Using compact lexical reordering table" << std::endl; + return new LexicalReorderingTableCompact(filePath, f_factors, e_factors, + c_factors); + } +#endif + return 0; +} + +void LexicalReorderingTableCompact::Load(std::string filePath) +{ + std::FILE* pFile = std::fopen(filePath.c_str(), "r"); + UTIL_THROW_IF2(pFile == NULL, "File " << filePath << " could not be opened"); + + //if(m_inMemory) + m_hash.Load(pFile); + //else + //m_hash.LoadIndex(pFile); + + size_t read = 0; + read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, + pFile); + read += std::fread(&m_multipleScoreTrees, sizeof(m_multipleScoreTrees), 1, + pFile); + + if (m_multipleScoreTrees) { + m_scoreTrees.resize(m_numScoreComponent); + for (size_t i = 0; i < m_numScoreComponent; i++) + m_scoreTrees[i] = new CanonicalHuffman<float>(pFile); + } + else { + m_scoreTrees.resize(1); + m_scoreTrees[0] = new CanonicalHuffman<float>(pFile); + } + + if (m_inMemory) m_scoresMemory.load(pFile, false); + else m_scoresMapped.load(pFile, true); +} + +} diff --git a/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.h b/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.h new file mode 100644 index 000000000..90abf4197 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/LexicalReorderingTableCompact.h @@ -0,0 +1,143 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_LexicalReorderingTableCompact_h +#define moses_LexicalReorderingTableCompact_h + +#include "BlockHashIndex.h" +#include "CanonicalHuffman.h" +#include "StringVector.h" +#include "../../TypeDef.h" +#include "../../Phrase.h" + +namespace Moses2 +{ + +//! additional types +class LexicalReorderingTable +{ +public: + LexicalReorderingTable(const FactorList& f_factors, + const FactorList& e_factors, const FactorList& c_factors) : + m_FactorsF(f_factors), m_FactorsE(e_factors), m_FactorsC(c_factors) + { + } + + virtual ~LexicalReorderingTable() + { + } + +public: + + virtual std::vector<float> + GetScore(const Phrase<Moses2::Word>& f, const Phrase<Moses2::Word>& e, const Phrase<Moses2::Word>& c) = 0; + + virtual + void InitializeForInput() + { + /* override for on-demand loading */ + } + ; + + virtual + void InitializeForInputPhrase(const Phrase<Moses2::Word>&) + { + } + + const FactorList& GetFFactorMask() const + { + return m_FactorsF; + } + const FactorList& GetEFactorMask() const + { + return m_FactorsE; + } + const FactorList& GetCFactorMask() const + { + return m_FactorsC; + } + + virtual + void DbgDump(std::ostream* out) const + { + *out << "Overwrite in subclass...\n"; + } + ; + // why is this not a pure virtual function? - UG + +protected: + FactorList m_FactorsF; + FactorList m_FactorsE; + FactorList m_FactorsC; +}; + +////////////////////////////////////////////////////////////////////////////////////////////// +class LexicalReorderingTableCompact: public LexicalReorderingTable +{ +private: + static bool s_inMemoryByDefault; + bool m_inMemory; + + size_t m_numScoreComponent; + bool m_multipleScoreTrees; + + BlockHashIndex m_hash; + + typedef CanonicalHuffman<float> ScoreTree; + std::vector<ScoreTree*> m_scoreTrees; + + StringVector<unsigned char, unsigned long, MmapAllocator> m_scoresMapped; + StringVector<unsigned char, unsigned long, std::allocator> m_scoresMemory; + + std::string MakeKey(const Phrase<Moses2::Word>& f, const Phrase<Moses2::Word>& e, const Phrase<Moses2::Word>& c) const; + std::string MakeKey(const std::string& f, const std::string& e, + const std::string& c) const; + +public: + LexicalReorderingTableCompact(const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); + + LexicalReorderingTableCompact(const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); + + virtual + ~LexicalReorderingTableCompact(); + + virtual std::vector<float> + GetScore(const Phrase<Moses2::Word>& f, const Phrase<Moses2::Word>& e, const Phrase<Moses2::Word>& c); + + static LexicalReorderingTable* + CheckAndLoad(const std::string& filePath, + const std::vector<FactorType>& f_factors, + const std::vector<FactorType>& e_factors, + const std::vector<FactorType>& c_factors); + + void + Load(std::string filePath); + +}; + +} + +#endif diff --git a/moses2/TranslationModel/CompactPT/ListCoders.h b/moses2/TranslationModel/CompactPT/ListCoders.h new file mode 100644 index 000000000..5a01274d9 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/ListCoders.h @@ -0,0 +1,394 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_ListCoders_h +#define moses_ListCoders_h + +#include <cmath> +#include <cassert> + +namespace Moses2 +{ + +template<typename T = unsigned int> +class VarIntType +{ +private: + template<typename IntType, typename OutIt> + static void EncodeSymbol(IntType input, OutIt output) + { + if (input == 0) { + *output = 0; + output++; + return; + } + + T msb = 1 << (sizeof(T) * 8 - 1); + IntType mask = ~msb; + IntType shift = (sizeof(T) * 8 - 1); + + while (input) { + T res = input & mask; + input >>= shift; + if (input) res |= msb; + *output = res; + output++; + } + } + ; + + template<typename InIt, typename IntType> + static void DecodeSymbol(InIt &it, InIt end, IntType &output) + { + T msb = 1 << (sizeof(T) * 8 - 1); + IntType shift = (sizeof(T) * 8 - 1); + + output = 0; + size_t i = 0; + while (it != end && *it & msb) { + IntType temp = *it & ~msb; + temp <<= shift * i; + output |= temp; + it++; + i++; + } + assert(it != end); + + IntType temp = *it; + temp <<= shift * i; + output |= temp; + it++; + } + +public: + + template<typename InIt, typename OutIt> + static void Encode(InIt it, InIt end, OutIt outIt) + { + while (it != end) { + EncodeSymbol(*it, outIt); + it++; + } + } + + template<typename InIt, typename OutIt> + static void Decode(InIt &it, InIt end, OutIt outIt) + { + while (it != end) { + size_t output; + DecodeSymbol(it, end, output); + *outIt = output; + outIt++; + } + } + + template<typename InIt> + static size_t DecodeAndSum(InIt &it, InIt end, size_t num) + { + size_t sum = 0; + size_t curr = 0; + + while (it != end && curr < num) { + size_t output; + DecodeSymbol(it, end, output); + sum += output; + curr++; + } + + return sum; + } + +}; + +typedef VarIntType<unsigned char> VarByte; + +typedef VarByte VarInt8; +typedef VarIntType<unsigned short> VarInt16; +typedef VarIntType<unsigned int> VarInt32; + +class Simple9 +{ +private: + typedef unsigned int uint; + + template<typename InIt> + inline static void EncodeSymbol(uint &output, InIt it, InIt end) + { + uint length = end - it; + + uint type = 0; + uint bitlength = 0; + + switch (length) { + case 1: + type = 1; + bitlength = 28; + break; + case 2: + type = 2; + bitlength = 14; + break; + case 3: + type = 3; + bitlength = 9; + break; + case 4: + type = 4; + bitlength = 7; + break; + case 5: + type = 5; + bitlength = 5; + break; + case 7: + type = 6; + bitlength = 4; + break; + case 9: + type = 7; + bitlength = 3; + break; + case 14: + type = 8; + bitlength = 2; + break; + case 28: + type = 9; + bitlength = 1; + break; + } + + output = 0; + output |= (type << 28); + + uint i = 0; + while (it != end) { + UTIL_THROW_IF2(*it > 268435455, + "You are trying to encode " << *it + << " with Simple9. Cannot encode numbers larger than 268435455 (2^28-1)"); + + uint l = bitlength * (length - i - 1); + output |= *it << l; + it++; + i++; + } + } + + template<typename OutIt> + static inline void DecodeSymbol(uint input, OutIt outIt) + { + uint type = (input >> 28); + + uint bitlen = 0; + uint shift = 0; + uint mask = 0; + + switch (type) { + case 1: + bitlen = 28; + shift = 0; + mask = 268435455; + break; + case 2: + bitlen = 14; + shift = 14; + mask = 16383; + break; + case 3: + bitlen = 9; + shift = 18; + mask = 511; + break; + case 4: + bitlen = 7; + shift = 21; + mask = 127; + break; + case 5: + bitlen = 5; + shift = 20; + mask = 31; + break; + case 6: + bitlen = 4; + shift = 24; + mask = 15; + break; + case 7: + bitlen = 3; + shift = 24; + mask = 7; + break; + case 8: + bitlen = 2; + shift = 26; + mask = 3; + break; + case 9: + bitlen = 1; + shift = 27; + mask = 1; + break; + } + + while (shift > 0) { + *outIt = (input >> shift) & mask; + shift -= bitlen; + outIt++; + } + *outIt = input & mask; + outIt++; + } + + static inline size_t DecodeAndSumSymbol(uint input, size_t num, size_t &curr) + { + uint type = (input >> 28); + + uint bitlen = 0; + uint shift = 0; + uint mask = 0; + + switch (type) { + case 1: + bitlen = 28; + shift = 0; + mask = 268435455; + break; + case 2: + bitlen = 14; + shift = 14; + mask = 16383; + break; + case 3: + bitlen = 9; + shift = 18; + mask = 511; + break; + case 4: + bitlen = 7; + shift = 21; + mask = 127; + break; + case 5: + bitlen = 5; + shift = 20; + mask = 31; + break; + case 6: + bitlen = 4; + shift = 24; + mask = 15; + break; + case 7: + bitlen = 3; + shift = 24; + mask = 7; + break; + case 8: + bitlen = 2; + shift = 26; + mask = 3; + break; + case 9: + bitlen = 1; + shift = 27; + mask = 1; + break; + } + + size_t sum = 0; + while (shift > 0) { + sum += (input >> shift) & mask; + shift -= bitlen; + if (++curr == num) return sum; + } + sum += input & mask; + curr++; + return sum; + } + +public: + template<typename InIt, typename OutIt> + static void Encode(InIt it, InIt end, OutIt outIt) + { + uint parts[] = { 1, 2, 3, 4, 5, 7, 9, 14, 28 }; + + uint buffer[28]; + for (InIt i = it; i < end; i++) { + uint lastbit = 1; + uint lastpos = 0; + uint lastyes = 0; + uint j = 0; + + double log2 = log(2); + while (j < 9 && lastpos < 28 && (i + lastpos) < end) { + if (lastpos >= parts[j]) j++; + + buffer[lastpos] = *(i + lastpos); + + uint reqbit = ceil(log(buffer[lastpos] + 1) / log2); + assert(reqbit <= 28); + + uint bit = 28 / floor(28 / reqbit); + if (lastbit < bit) lastbit = bit; + + if (parts[j] > 28 / lastbit) break; + else if (lastpos == parts[j] - 1) lastyes = lastpos; + + lastpos++; + } + i += lastyes; + + uint length = lastyes + 1; + uint output; + EncodeSymbol(output, buffer, buffer + length); + + *outIt = output; + outIt++; + } + } + + template<typename InIt, typename OutIt> + static void Decode(InIt &it, InIt end, OutIt outIt) + { + while (it != end) { + DecodeSymbol(*it, outIt); + it++; + } + } + + template<typename InIt> + static size_t DecodeAndSum(InIt &it, InIt end, size_t num) + { + size_t sum = 0; + size_t curr = 0; + while (it != end && curr < num) { + sum += DecodeAndSumSymbol(*it, num, curr); + it++; + } + assert(curr == num); + return sum; + } +}; + +} + +#endif diff --git a/moses2/TranslationModel/CompactPT/MmapAllocator.h b/moses2/TranslationModel/CompactPT/MmapAllocator.h new file mode 100644 index 000000000..1e40d8d41 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/MmapAllocator.h @@ -0,0 +1,217 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_MmapAllocator_h +#define moses_MmapAllocator_h + +#include <limits> +#include <iostream> +#include <cstdio> +#include <unistd.h> + +#if defined(_WIN32) || defined(_WIN64) +#include <windows.h> +#include <io.h> +#else +#include <sys/mman.h> +#endif + +#include "util/mmap.hh" + +namespace Moses2 +{ +template<class T> +class MmapAllocator +{ +protected: + std::FILE* m_file_ptr; + size_t m_file_desc; + + size_t m_page_size; + size_t m_map_size; + + char* m_data_ptr; + size_t m_data_offset; + bool m_fixed; + size_t* m_count; + +public: + typedef T value_type; + typedef T* pointer; + typedef const T* const_pointer; + typedef T& reference; + typedef const T& const_reference; + typedef std::size_t size_type; + typedef std::ptrdiff_t difference_type; + + MmapAllocator() throw () : + m_file_ptr(std::tmpfile()), m_file_desc(fileno(m_file_ptr)), m_page_size( + util::SizePage()), m_map_size(0), m_data_ptr(0), m_data_offset(0), m_fixed( + false), m_count(new size_t(0)) + { + } + + MmapAllocator(std::FILE* f_ptr) throw () : + m_file_ptr(f_ptr), m_file_desc(fileno(m_file_ptr)), m_page_size( + util::SizePage()), m_map_size(0), m_data_ptr(0), m_data_offset(0), m_fixed( + false), m_count(new size_t(0)) + { + } + + MmapAllocator(std::FILE* f_ptr, size_t data_offset) throw () : + m_file_ptr(f_ptr), m_file_desc(fileno(m_file_ptr)), m_page_size( + util::SizePage()), m_map_size(0), m_data_ptr(0), m_data_offset( + data_offset), m_fixed(true), m_count(new size_t(0)) + { + } + + MmapAllocator(std::string fileName) throw () : + m_file_ptr(std::fopen(fileName.c_str(), "wb+")), m_file_desc( + fileno(m_file_ptr)), m_page_size(util::SizePage()), m_map_size(0), m_data_ptr( + 0), m_data_offset(0), m_fixed(false), m_count(new size_t(0)) + { + } + + MmapAllocator(const MmapAllocator& c) throw () : + m_file_ptr(c.m_file_ptr), m_file_desc(c.m_file_desc), m_page_size( + c.m_page_size), m_map_size(c.m_map_size), m_data_ptr(c.m_data_ptr), m_data_offset( + c.m_data_offset), m_fixed(c.m_fixed), m_count(c.m_count) + { + (*m_count)++; + } + + ~MmapAllocator() throw () + { + if (m_data_ptr && *m_count == 0) { + util::UnmapOrThrow(m_data_ptr, m_map_size); + if (!m_fixed && std::ftell(m_file_ptr) != -1) std::fclose(m_file_ptr); + } + (*m_count)--; + } + + template<class U> + struct rebind + { + typedef MmapAllocator<U> other; + }; + + pointer address(reference value) const + { + return &value; + } + + const_pointer address(const_reference value) const + { + return &value; + } + + size_type max_size() const throw () + { + return std::numeric_limits<size_t>::max() / sizeof(value_type); + } + + pointer allocate(size_type num, const void* = 0) + { + m_map_size = num * sizeof(T); + +#if defined(_WIN32) || defined(_WIN64) + // On Windows, MAP_SHARED is not defined and MapOrThrow ignores the flags. + const int map_shared = 0; +#else + const int map_shared = MAP_SHARED; +#endif + if (!m_fixed) { + size_t read = 0; + read += ftruncate(m_file_desc, m_map_size); + m_data_ptr = (char *) util::MapOrThrow(m_map_size, true, map_shared, + false, m_file_desc, 0); + return (pointer) m_data_ptr; + } + else { + const size_t map_offset = (m_data_offset / m_page_size) * m_page_size; + const size_t relative_offset = m_data_offset - map_offset; + const size_t adjusted_map_size = m_map_size + relative_offset; + + m_data_ptr = (char *) util::MapOrThrow(adjusted_map_size, false, + map_shared, false, m_file_desc, map_offset); + + return (pointer) (m_data_ptr + relative_offset); + } + } + + void deallocate(pointer p, size_type num) + { + if (!m_fixed) { + util::UnmapOrThrow(p, num * sizeof(T)); + } + else { + const size_t map_offset = (m_data_offset / m_page_size) * m_page_size; + const size_t relative_offset = m_data_offset - map_offset; + const size_t adjusted_map_size = m_map_size + relative_offset; + + util::UnmapOrThrow((pointer) ((char*) p - relative_offset), + adjusted_map_size); + } + } + + void construct(pointer p, const T& value) + { + if (!m_fixed) new (p) value_type(value); + } + void destroy(pointer p) + { + if (!m_fixed) p->~T(); + } + + template<class T1, class T2> + friend bool operator==(const MmapAllocator<T1>&, + const MmapAllocator<T2>&) throw (); + + template<class T1, class T2> + friend bool operator!=(const MmapAllocator<T1>&, + const MmapAllocator<T2>&) throw (); +}; + +template<class T1, class T2> +bool operator==(const MmapAllocator<T1>& a1, + const MmapAllocator<T2>& a2) throw () +{ + bool equal = true; + equal &= a1.m_file_ptr == a2.m_file_ptr; + equal &= a1.m_file_desc == a2.m_file_desc; + equal &= a1.m_page_size == a2.m_page_size; + equal &= a1.m_map_size == a2.m_map_size; + equal &= a1.m_data_ptr == a2.m_data_ptr; + equal &= a1.m_data_offset == a2.m_data_offset; + equal &= a1.m_fixed == a2.m_fixed; + return equal; +} + +template<class T1, class T2> +bool operator!=(const MmapAllocator<T1>& a1, + const MmapAllocator<T2>& a2) throw () +{ + return !(a1 == a2); +} + +} + +#endif diff --git a/moses2/TranslationModel/CompactPT/MonotonicVector.h b/moses2/TranslationModel/CompactPT/MonotonicVector.h new file mode 100644 index 000000000..586397db8 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/MonotonicVector.h @@ -0,0 +1,247 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_MonotonicVector_h +#define moses_MonotonicVector_h + +// MonotonicVector - Represents a monotonic increasing function that maps +// positive integers of any size onto a given number type. Each value has to be +// equal or larger than the previous one. Depending on the stepSize it can save +// up to 90% of memory compared to a std::vector<long>. Time complexity is roughly +// constant, in the worst case, however, stepSize times slower than a normal +// std::vector. + +#include <vector> +#include <limits> +#include <algorithm> +#include <cstdio> +#include <cassert> + +#include "ThrowingFwrite.h" +#include "ListCoders.h" +#include "MmapAllocator.h" + +namespace Moses2 +{ + +template<typename PosT = size_t, typename NumT = size_t, PosT stepSize = 32, + template<typename > class Allocator = std::allocator> +class MonotonicVector +{ +private: + typedef std::vector<NumT, Allocator<NumT> > Anchors; + typedef std::vector<unsigned int, Allocator<unsigned int> > Diffs; + + Anchors m_anchors; + Diffs m_diffs; + std::vector<unsigned int> m_tempDiffs; + + size_t m_size; + PosT m_last; + bool m_final; + +public: + typedef PosT value_type; + + MonotonicVector() : + m_size(0), m_last(0), m_final(false) + { + } + + size_t size() const + { + return m_size + m_tempDiffs.size(); + } + + PosT at(size_t i) const + { + PosT s = stepSize; + PosT j = m_anchors[i / s]; + PosT r = i % s; + + typename Diffs::const_iterator it = m_diffs.begin() + j; + + PosT k = 0; + k += VarInt32::DecodeAndSum(it, m_diffs.end(), 1); + if (i < m_size) k += Simple9::DecodeAndSum(it, m_diffs.end(), r); + else if (i < m_size + m_tempDiffs.size()) for (size_t l = 0; l < r; l++) + k += m_tempDiffs[l]; + + return k; + } + + PosT operator[](PosT i) const + { + return at(i); + } + + PosT back() const + { + return at(size() - 1); + } + + void push_back(PosT i) + { + assert(m_final != true); + + if (m_anchors.size() == 0 && m_tempDiffs.size() == 0) { + m_anchors.push_back(0); + VarInt32::Encode(&i, &i + 1, std::back_inserter(m_diffs)); + m_last = i; + m_size++; + + return; + } + + if (m_tempDiffs.size() == stepSize - 1) { + Simple9::Encode(m_tempDiffs.begin(), m_tempDiffs.end(), + std::back_inserter(m_diffs)); + m_anchors.push_back(m_diffs.size()); + VarInt32::Encode(&i, &i + 1, std::back_inserter(m_diffs)); + + m_size += m_tempDiffs.size() + 1; + m_tempDiffs.clear(); + } + else { + PosT last = m_last; + PosT diff = i - last; + m_tempDiffs.push_back(diff); + } + m_last = i; + } + + void commit() + { + assert(m_final != true); + Simple9::Encode(m_tempDiffs.begin(), m_tempDiffs.end(), + std::back_inserter(m_diffs)); + m_size += m_tempDiffs.size(); + m_tempDiffs.clear(); + m_final = true; + } + + size_t usage() + { + return m_diffs.size() * sizeof(unsigned int) + + m_anchors.size() * sizeof(NumT); + } + + size_t load(std::FILE* in, bool map = false) + { + size_t byteSize = 0; + + byteSize += fread(&m_final, sizeof(bool), 1, in) * sizeof(bool); + byteSize += fread(&m_size, sizeof(size_t), 1, in) * sizeof(size_t); + byteSize += fread(&m_last, sizeof(PosT), 1, in) * sizeof(PosT); + + byteSize += loadVector(m_diffs, in, map); + byteSize += loadVector(m_anchors, in, map); + + return byteSize; + } + + template<typename ValueT> + size_t loadVector(std::vector<ValueT, std::allocator<ValueT> >& v, + std::FILE* in, bool map = false) + { + // Can only be read into memory. Mapping not possible with std:allocator. + assert(map == false); + + size_t byteSize = 0; + + size_t valSize; + byteSize += std::fread(&valSize, sizeof(size_t), 1, in) * sizeof(size_t); + + v.resize(valSize, 0); + byteSize += std::fread(&v[0], sizeof(ValueT), valSize, in) * sizeof(ValueT); + + return byteSize; + } + + template<typename ValueT> + size_t loadVector(std::vector<ValueT, MmapAllocator<ValueT> >& v, + std::FILE* in, bool map = false) + { + size_t byteSize = 0; + + size_t valSize; + byteSize += std::fread(&valSize, sizeof(size_t), 1, in) * sizeof(size_t); + + if (map == false) { + // Read data into temporary file (default constructor of MmapAllocator) + // and map memory onto temporary file. Can be resized. + + v.resize(valSize, 0); + byteSize += std::fread(&v[0], sizeof(ValueT), valSize, in) + * sizeof(ValueT); + } + else { + // Map it directly on specified region of file "in" starting at valPos + // with length valSize * sizeof(ValueT). Mapped region cannot be resized. + + size_t valPos = std::ftell(in); + + Allocator<ValueT> alloc(in, valPos); + std::vector<ValueT, Allocator<ValueT> > vTemp(alloc); + vTemp.resize(valSize); + v.swap(vTemp); + + std::fseek(in, valSize * sizeof(ValueT), SEEK_CUR); + byteSize += valSize * sizeof(ValueT); + } + + return byteSize; + } + + size_t save(std::FILE* out) + { + if (!m_final) commit(); + + bool byteSize = 0; + byteSize += ThrowingFwrite(&m_final, sizeof(bool), 1, out) * sizeof(bool); + byteSize += ThrowingFwrite(&m_size, sizeof(size_t), 1, out) + * sizeof(size_t); + byteSize += ThrowingFwrite(&m_last, sizeof(PosT), 1, out) * sizeof(PosT); + + size_t size = m_diffs.size(); + byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t); + byteSize += ThrowingFwrite(&m_diffs[0], sizeof(unsigned int), size, out) + * sizeof(unsigned int); + + size = m_anchors.size(); + byteSize += ThrowingFwrite(&size, sizeof(size_t), 1, out) * sizeof(size_t); + byteSize += ThrowingFwrite(&m_anchors[0], sizeof(NumT), size, out) + * sizeof(NumT); + + return byteSize; + } + + void swap(MonotonicVector<PosT, NumT, stepSize, Allocator> &mv) + { + if (!m_final) commit(); + + m_diffs.swap(mv.m_diffs); + m_anchors.swap(mv.m_anchors); + } +}; + +} +#endif diff --git a/moses2/TranslationModel/CompactPT/MurmurHash3.cpp b/moses2/TranslationModel/CompactPT/MurmurHash3.cpp new file mode 100644 index 000000000..c3e567af6 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/MurmurHash3.cpp @@ -0,0 +1,424 @@ +//----------------------------------------------------------------------------- +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +// Note - The x86 and x64 versions do _not_ produce the same results, as the +// algorithms are optimized for their respective platforms. You can still +// compile and run any of them on any platform, but your performance with the +// non-native version will be less than optimal. + +#include "MurmurHash3.h" + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +// Microsoft Visual Studio + +#if defined(_MSC_VER) + +#define FORCE_INLINE __forceinline + +#include <cstdlib> + +#define ROTL32(x,y) _rotl(x,y) +#define ROTL64(x,y) _rotl64(x,y) + +#define BIG_CONSTANT(x) (x) + +// Other compilers + +#else // defined(_MSC_VER) + +#define FORCE_INLINE inline __attribute__((always_inline)) + +inline uint32_t rotl32(uint32_t x, int8_t r) +{ + return (x << r) | (x >> (32 - r)); +} + +inline uint64_t rotl64(uint64_t x, int8_t r) +{ + return (x << r) | (x >> (64 - r)); +} + +#define ROTL32(x,y) rotl32(x,y) +#define ROTL64(x,y) rotl64(x,y) + +#define BIG_CONSTANT(x) (x##LLU) + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- +// Block read - if your platform needs to do endian-swapping or can only +// handle aligned reads, do the conversion here + +FORCE_INLINE uint32_t getblock(const uint32_t * p, int i) +{ + return p[i]; +} + +FORCE_INLINE uint64_t getblock(const uint64_t * p, int i) +{ + return p[i]; +} + +//----------------------------------------------------------------------------- +// Finalization mix - force all bits of a hash block to avalanche + +FORCE_INLINE uint32_t fmix(uint32_t h) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + + return h; +} + +//---------- + +FORCE_INLINE uint64_t fmix(uint64_t k) +{ + k ^= k >> 33; + k *= BIG_CONSTANT(0xff51afd7ed558ccd); + k ^= k >> 33; + k *= BIG_CONSTANT(0xc4ceb9fe1a85ec53); + k ^= k >> 33; + + return k; +} + +//----------------------------------------------------------------------------- + +void MurmurHash3_x86_32(const void * key, int len, uint32_t seed, void * out) +{ + const uint8_t * data = (const uint8_t*) key; + const int nblocks = len / 4; + + uint32_t h1 = seed; + + uint32_t c1 = 0xcc9e2d51; + uint32_t c2 = 0x1b873593; + + //---------- + // body + + const uint32_t * blocks = (const uint32_t *) (data + nblocks * 4); + + for (int i = -nblocks; i; i++) { + uint32_t k1 = getblock(blocks, i); + + k1 *= c1; + k1 = ROTL32(k1, 15); + k1 *= c2; + + h1 ^= k1; + h1 = ROTL32(h1, 13); + h1 = h1 * 5 + 0xe6546b64; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*) (data + nblocks * 4); + + uint32_t k1 = 0; + + switch (len & 3) { + case 3: + k1 ^= tail[2] << 16; + case 2: + k1 ^= tail[1] << 8; + case 1: + k1 ^= tail[0]; + k1 *= c1; + k1 = ROTL32(k1, 15); + k1 *= c2; + h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + + h1 = fmix(h1); + + *(uint32_t*) out = h1; +} + +//----------------------------------------------------------------------------- + +void MurmurHash3_x86_128(const void * key, const int len, uint32_t seed, + void * out) +{ + const uint8_t * data = (const uint8_t*) key; + const int nblocks = len / 16; + + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; + + uint32_t c1 = 0x239b961b; + uint32_t c2 = 0xab0e9789; + uint32_t c3 = 0x38b34ae5; + uint32_t c4 = 0xa1e38b93; + + //---------- + // body + + const uint32_t * blocks = (const uint32_t *) (data + nblocks * 16); + + for (int i = -nblocks; i; i++) { + uint32_t k1 = getblock(blocks, i * 4 + 0); + uint32_t k2 = getblock(blocks, i * 4 + 1); + uint32_t k3 = getblock(blocks, i * 4 + 2); + uint32_t k4 = getblock(blocks, i * 4 + 3); + + k1 *= c1; + k1 = ROTL32(k1, 15); + k1 *= c2; + h1 ^= k1; + + h1 = ROTL32(h1, 19); + h1 += h2; + h1 = h1 * 5 + 0x561ccd1b; + + k2 *= c2; + k2 = ROTL32(k2, 16); + k2 *= c3; + h2 ^= k2; + + h2 = ROTL32(h2, 17); + h2 += h3; + h2 = h2 * 5 + 0x0bcaa747; + + k3 *= c3; + k3 = ROTL32(k3, 17); + k3 *= c4; + h3 ^= k3; + + h3 = ROTL32(h3, 15); + h3 += h4; + h3 = h3 * 5 + 0x96cd1c35; + + k4 *= c4; + k4 = ROTL32(k4, 18); + k4 *= c1; + h4 ^= k4; + + h4 = ROTL32(h4, 13); + h4 += h1; + h4 = h4 * 5 + 0x32ac3b17; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*) (data + nblocks * 16); + + uint32_t k1 = 0; + uint32_t k2 = 0; + uint32_t k3 = 0; + uint32_t k4 = 0; + + switch (len & 15) { + case 15: + k4 ^= tail[14] << 16; + case 14: + k4 ^= tail[13] << 8; + case 13: + k4 ^= tail[12] << 0; + k4 *= c4; + k4 = ROTL32(k4, 18); + k4 *= c1; + h4 ^= k4; + + case 12: + k3 ^= tail[11] << 24; + case 11: + k3 ^= tail[10] << 16; + case 10: + k3 ^= tail[9] << 8; + case 9: + k3 ^= tail[8] << 0; + k3 *= c3; + k3 = ROTL32(k3, 17); + k3 *= c4; + h3 ^= k3; + + case 8: + k2 ^= tail[7] << 24; + case 7: + k2 ^= tail[6] << 16; + case 6: + k2 ^= tail[5] << 8; + case 5: + k2 ^= tail[4] << 0; + k2 *= c2; + k2 = ROTL32(k2, 16); + k2 *= c3; + h2 ^= k2; + + case 4: + k1 ^= tail[3] << 24; + case 3: + k1 ^= tail[2] << 16; + case 2: + k1 ^= tail[1] << 8; + case 1: + k1 ^= tail[0] << 0; + k1 *= c1; + k1 = ROTL32(k1, 15); + k1 *= c2; + h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + h2 ^= len; + h3 ^= len; + h4 ^= len; + + h1 += h2; + h1 += h3; + h1 += h4; + h2 += h1; + h3 += h1; + h4 += h1; + + h1 = fmix(h1); + h2 = fmix(h2); + h3 = fmix(h3); + h4 = fmix(h4); + + h1 += h2; + h1 += h3; + h1 += h4; + h2 += h1; + h3 += h1; + h4 += h1; + + ((uint32_t*) out)[0] = h1; + ((uint32_t*) out)[1] = h2; + ((uint32_t*) out)[2] = h3; + ((uint32_t*) out)[3] = h4; +} + +//----------------------------------------------------------------------------- + +void MurmurHash3_x64_128(const void * key, const int len, const uint32_t seed, + void * out) +{ + const uint8_t * data = (const uint8_t*) key; + const int nblocks = len / 16; + + uint64_t h1 = seed; + uint64_t h2 = seed; + + uint64_t c1 = BIG_CONSTANT(0x87c37b91114253d5); + uint64_t c2 = BIG_CONSTANT(0x4cf5ad432745937f); + + //---------- + // body + + const uint64_t * blocks = (const uint64_t *) (data); + + for (int i = 0; i < nblocks; i++) { + uint64_t k1 = getblock(blocks, i * 2 + 0); + uint64_t k2 = getblock(blocks, i * 2 + 1); + + k1 *= c1; + k1 = ROTL64(k1, 31); + k1 *= c2; + h1 ^= k1; + + h1 = ROTL64(h1, 27); + h1 += h2; + h1 = h1 * 5 + 0x52dce729; + + k2 *= c2; + k2 = ROTL64(k2, 33); + k2 *= c1; + h2 ^= k2; + + h2 = ROTL64(h2, 31); + h2 += h1; + h2 = h2 * 5 + 0x38495ab5; + } + + //---------- + // tail + + const uint8_t * tail = (const uint8_t*) (data + nblocks * 16); + + uint64_t k1 = 0; + uint64_t k2 = 0; + + switch (len & 15) { + case 15: + k2 ^= uint64_t(tail[14]) << 48; + case 14: + k2 ^= uint64_t(tail[13]) << 40; + case 13: + k2 ^= uint64_t(tail[12]) << 32; + case 12: + k2 ^= uint64_t(tail[11]) << 24; + case 11: + k2 ^= uint64_t(tail[10]) << 16; + case 10: + k2 ^= uint64_t(tail[9]) << 8; + case 9: + k2 ^= uint64_t(tail[8]) << 0; + k2 *= c2; + k2 = ROTL64(k2, 33); + k2 *= c1; + h2 ^= k2; + + case 8: + k1 ^= uint64_t(tail[7]) << 56; + case 7: + k1 ^= uint64_t(tail[6]) << 48; + case 6: + k1 ^= uint64_t(tail[5]) << 40; + case 5: + k1 ^= uint64_t(tail[4]) << 32; + case 4: + k1 ^= uint64_t(tail[3]) << 24; + case 3: + k1 ^= uint64_t(tail[2]) << 16; + case 2: + k1 ^= uint64_t(tail[1]) << 8; + case 1: + k1 ^= uint64_t(tail[0]) << 0; + k1 *= c1; + k1 = ROTL64(k1, 31); + k1 *= c2; + h1 ^= k1; + }; + + //---------- + // finalization + + h1 ^= len; + h2 ^= len; + + h1 += h2; + h2 += h1; + + h1 = fmix(h1); + h2 = fmix(h2); + + h1 += h2; + h2 += h1; + + ((uint64_t*) out)[0] = h1; + ((uint64_t*) out)[1] = h2; +} + +//----------------------------------------------------------------------------- + diff --git a/moses2/TranslationModel/CompactPT/MurmurHash3.h b/moses2/TranslationModel/CompactPT/MurmurHash3.h new file mode 100644 index 000000000..f513008cf --- /dev/null +++ b/moses2/TranslationModel/CompactPT/MurmurHash3.h @@ -0,0 +1,37 @@ +//----------------------------------------------------------------------------- +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +#ifndef _MURMURHASH3_H_ +#define _MURMURHASH3_H_ + +//----------------------------------------------------------------------------- +// Platform-specific functions and macros + +// Microsoft Visual Studio + +#if defined(_MSC_VER) + +typedef unsigned char uint8_t; +typedef unsigned long uint32_t; +typedef unsigned __int64 uint64_t; + +// Other compilers + +#else // defined(_MSC_VER) + +#include <stdint.h> + +#endif // !defined(_MSC_VER) + +//----------------------------------------------------------------------------- + +void MurmurHash3_x86_32(const void * key, int len, uint32_t seed, void * out); + +void MurmurHash3_x86_128(const void * key, int len, uint32_t seed, void * out); + +void MurmurHash3_x64_128(const void * key, int len, uint32_t seed, void * out); + +//----------------------------------------------------------------------------- + +#endif // _MURMURHASH3_H_ diff --git a/moses2/TranslationModel/CompactPT/PackedArray.h b/moses2/TranslationModel/CompactPT/PackedArray.h new file mode 100644 index 000000000..409c3cca8 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/PackedArray.h @@ -0,0 +1,207 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_PackedArray_h +#define moses_PackedArray_h + +#include <vector> +#include <cmath> +#include <cstring> +#include <cstdio> + +#include "ThrowingFwrite.h" + +namespace Moses2 +{ + +template<typename T = size_t, typename D = unsigned char> +class PackedArray +{ +protected: + static size_t m_dataBits; + + size_t m_size; + size_t m_storageSize; + D* m_storage; + +public: + PackedArray() + { + m_size = 0; + m_storageSize = 0; + m_storage = new D[0]; + } + + PackedArray(size_t size, size_t bits) : + m_size(size) + { + m_storageSize = ceil(float(bits * size) / float(m_dataBits)); + m_storage = new D[m_storageSize]; + } + + PackedArray(const PackedArray<T, D> &c) + { + m_size = c.m_size; + + m_storageSize = c.m_storageSize; + m_storage = new D[m_storageSize]; + + std::memcpy(m_storage, c.m_storage, m_storageSize * sizeof(D)); + } + + virtual ~PackedArray() + { + delete[] m_storage; + m_size = 0; + m_storageSize = 0; + m_storage = 0; + } + + T Get(size_t i, size_t bits) const + { + T out = 0; + + size_t bitstart = (i * bits); + size_t bitpos = bitstart; + + size_t zero = ((1ul << (bits)) - 1); + + while (bitpos - bitstart < bits) { + size_t pos = bitpos / m_dataBits; + size_t off = bitpos % m_dataBits; + + out |= (T(m_storage[pos]) << (bitpos - bitstart)) >> off; + + bitpos += (m_dataBits - off); + } + + out &= zero; + return out; + } + + void Set(size_t i, T v, size_t bits) + { + size_t bitstart = (i * bits); + size_t bitpos = bitstart; + + while (bitpos - bitstart < bits) { + size_t pos = bitpos / m_dataBits; + size_t off = bitpos % m_dataBits; + + size_t rest = bits - (bitpos - bitstart); + D zero = ~((1ul << (rest + off)) - 1) | ((1ul << off) - 1); + + m_storage[pos] &= zero; + m_storage[pos] |= v << off; + v = v >> (m_dataBits - off); + bitpos += (m_dataBits - off); + } + } + + virtual D*& GetStorage() + { + return m_storage; + } + + virtual size_t GetStorageSize() const + { + return m_storageSize; + } + + virtual size_t Size() const + { + return m_size; + } + + virtual size_t Load(std::FILE* in) + { + size_t a1 = std::ftell(in); + + size_t read = 0; + read += std::fread(&m_size, sizeof(m_size), 1, in); + read += std::fread(&m_storageSize, sizeof(m_storageSize), 1, in); + delete[] m_storage; + m_storage = new D[m_storageSize]; + read += std::fread(m_storage, sizeof(D), m_storageSize, in); + + size_t a2 = std::ftell(in); + return a2 - a1; + } + + virtual size_t Save(std::FILE* out) + { + size_t a1 = std::ftell(out); + + ThrowingFwrite(&m_size, sizeof(m_size), 1, out); + ThrowingFwrite(&m_storageSize, sizeof(m_storageSize), 1, out); + ThrowingFwrite(m_storage, sizeof(D), m_storageSize, out); + + size_t a2 = std::ftell(out); + return a2 - a1; + } + +}; + +template<typename T, typename D> +size_t PackedArray<T, D>::m_dataBits = sizeof(D) * 8; + +/**************************************************************************/ + +template<typename T = size_t, typename D = unsigned char> +class PairedPackedArray: public PackedArray<T, D> +{ +public: + PairedPackedArray() : + PackedArray<T, D>() + { + } + + PairedPackedArray(size_t size, size_t bits1, size_t bits2) : + PackedArray<T, D>(size, bits1 + bits2) + { + } + + void Set(size_t i, T a, T b, size_t bits1, size_t bits2) + { + T c = 0; + c = a | (b << bits1); + PackedArray<T, D>::Set(i, c, bits1 + bits2); + } + + void Set(size_t i, std::pair<T, T> p, size_t bits1, size_t bits2) + { + T c = 0; + c = p.second | (p.first << bits1); + PackedArray<T, D>::Set(i, c); + } + + std::pair<T, T> Get(size_t i, size_t bits1, size_t bits2) + { + T v = PackedArray<T, D>::Get(i, bits1 + bits2); + T a = v & ((1 << bits1) - 1); + T b = v >> bits1; + return std::pair<T, T>(a, b); + } +}; + +} + +#endif diff --git a/moses2/TranslationModel/CompactPT/PhraseDecoder.cpp b/moses2/TranslationModel/CompactPT/PhraseDecoder.cpp new file mode 100644 index 000000000..7860fed94 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/PhraseDecoder.cpp @@ -0,0 +1,466 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2006 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include <deque> + +#include "PhraseDecoder.h" +#include "../../System.h" +#include "../../SubPhrase.h" + +using namespace std; + +namespace Moses2 +{ + +PhraseDecoder::PhraseDecoder( + PhraseTableCompact &phraseDictionary, + const std::vector<FactorType>* input, + const std::vector<FactorType>* output, + size_t numScoreComponent + // , const std::vector<float>* weight +) + : m_coding(None), m_numScoreComponent(numScoreComponent), + m_containsAlignmentInfo(true), m_maxRank(0), + m_symbolTree(0), m_multipleScoreTrees(false), + m_scoreTrees(1), m_alignTree(0), + m_phraseDictionary(phraseDictionary), m_input(input), m_output(output), + // m_weight(weight), + m_separator(" ||| ") +{ } + +PhraseDecoder::~PhraseDecoder() +{ + if(m_symbolTree) + delete m_symbolTree; + + for(size_t i = 0; i < m_scoreTrees.size(); i++) + if(m_scoreTrees[i]) + delete m_scoreTrees[i]; + + if(m_alignTree) + delete m_alignTree; +} + +inline unsigned PhraseDecoder::GetSourceSymbolId(std::string& symbol) +{ + boost::unordered_map<std::string, unsigned>::iterator it + = m_sourceSymbolsMap.find(symbol); + if(it != m_sourceSymbolsMap.end()) + return it->second; + + size_t idx = m_sourceSymbols.find(symbol); + m_sourceSymbolsMap[symbol] = idx; + return idx; +} + +inline std::string PhraseDecoder::GetTargetSymbol(unsigned idx) const +{ + if(idx < m_targetSymbols.size()) + return m_targetSymbols[idx]; + return std::string("##ERROR##"); +} + +inline size_t PhraseDecoder::GetREncType(unsigned encodedSymbol) +{ + return (encodedSymbol >> 30) + 1; +} + +inline size_t PhraseDecoder::GetPREncType(unsigned encodedSymbol) +{ + return (encodedSymbol >> 31) + 1; +} + +inline unsigned PhraseDecoder::GetTranslation(unsigned srcIdx, size_t rank) +{ + size_t srcTrgIdx = m_lexicalTableIndex[srcIdx]; + return m_lexicalTable[srcTrgIdx + rank].second; +} + +size_t PhraseDecoder::GetMaxSourcePhraseLength() +{ + return m_maxPhraseLength; +} + +inline unsigned PhraseDecoder::DecodeREncSymbol1(unsigned encodedSymbol) +{ + return encodedSymbol &= ~(3 << 30); +} + +inline unsigned PhraseDecoder::DecodeREncSymbol2Rank(unsigned encodedSymbol) +{ + return encodedSymbol &= ~(255 << 24); +} + +inline unsigned PhraseDecoder::DecodeREncSymbol2Position(unsigned encodedSymbol) +{ + encodedSymbol &= ~(3 << 30); + encodedSymbol >>= 24; + return encodedSymbol; +} + +inline unsigned PhraseDecoder::DecodeREncSymbol3(unsigned encodedSymbol) +{ + return encodedSymbol &= ~(3 << 30); +} + +inline unsigned PhraseDecoder::DecodePREncSymbol1(unsigned encodedSymbol) +{ + return encodedSymbol &= ~(1 << 31); +} + +inline int PhraseDecoder::DecodePREncSymbol2Left(unsigned encodedSymbol) +{ + return ((encodedSymbol >> 25) & 63) - 32; +} + +inline int PhraseDecoder::DecodePREncSymbol2Right(unsigned encodedSymbol) +{ + return ((encodedSymbol >> 19) & 63) - 32; +} + +inline unsigned PhraseDecoder::DecodePREncSymbol2Rank(unsigned encodedSymbol) +{ + return (encodedSymbol & 524287); +} + +size_t PhraseDecoder::Load(std::FILE* in) +{ + size_t start = std::ftell(in); + size_t read = 0; + + read += std::fread(&m_coding, sizeof(m_coding), 1, in); + read += std::fread(&m_numScoreComponent, sizeof(m_numScoreComponent), 1, in); + read += std::fread(&m_containsAlignmentInfo, sizeof(m_containsAlignmentInfo), 1, in); + read += std::fread(&m_maxRank, sizeof(m_maxRank), 1, in); + read += std::fread(&m_maxPhraseLength, sizeof(m_maxPhraseLength), 1, in); + + if(m_coding == REnc) { + m_sourceSymbols.load(in); + + size_t size; + read += std::fread(&size, sizeof(size_t), 1, in); + m_lexicalTableIndex.resize(size); + read += std::fread(&m_lexicalTableIndex[0], sizeof(size_t), size, in); + + read += std::fread(&size, sizeof(size_t), 1, in); + m_lexicalTable.resize(size); + read += std::fread(&m_lexicalTable[0], sizeof(SrcTrg), size, in); + } + + m_targetSymbols.load(in); + + m_symbolTree = new CanonicalHuffman<unsigned>(in); + + read += std::fread(&m_multipleScoreTrees, sizeof(m_multipleScoreTrees), 1, in); + if(m_multipleScoreTrees) { + m_scoreTrees.resize(m_numScoreComponent); + for(size_t i = 0; i < m_numScoreComponent; i++) + m_scoreTrees[i] = new CanonicalHuffman<float>(in); + } else { + m_scoreTrees.resize(1); + m_scoreTrees[0] = new CanonicalHuffman<float>(in); + } + + if(m_containsAlignmentInfo) + m_alignTree = new CanonicalHuffman<AlignPoint>(in); + + size_t end = std::ftell(in); + return end - start; +} + +std::string PhraseDecoder::MakeSourceKey(std::string &source) +{ + return source + m_separator; +} + +TargetPhraseVectorPtr PhraseDecoder::CreateTargetPhraseCollection( + const ManagerBase &mgr, + const Phrase<Word> &sourcePhrase, + bool topLevel, + bool eval) +{ + + // Not using TargetPhraseCollection avoiding "new" operator + // which can introduce heavy locking with multiple threads + TargetPhraseVectorPtr tpv(new TargetPhraseVector()); + size_t bitsLeft = 0; + + if(m_coding == PREnc) { + std::pair<TargetPhraseVectorPtr, size_t> cachedPhraseColl + = m_decodingCache.Retrieve(sourcePhrase); + + // Has been cached and is complete or does not need to be completed + if(cachedPhraseColl.first != NULL && (!topLevel || cachedPhraseColl.second == 0)) + return cachedPhraseColl.first; + + // Has been cached, but is incomplete + else if(cachedPhraseColl.first != NULL) { + bitsLeft = cachedPhraseColl.second; + tpv->resize(cachedPhraseColl.first->size()); + std::copy(cachedPhraseColl.first->begin(), + cachedPhraseColl.first->end(), + tpv->begin()); + } + } + + // Retrieve source phrase identifier + std::string sourcePhraseString = sourcePhrase.GetString(*m_input); + size_t sourcePhraseId = m_phraseDictionary.m_hash[MakeSourceKey(sourcePhraseString)]; + /* + cerr << "sourcePhraseString=" << sourcePhraseString << " " + << sourcePhraseId + << endl; + */ + if(sourcePhraseId != m_phraseDictionary.m_hash.GetSize()) { + // Retrieve compressed and encoded target phrase collection + std::string encodedPhraseCollection; + if(m_phraseDictionary.m_inMemory) + encodedPhraseCollection = m_phraseDictionary.m_targetPhrasesMemory[sourcePhraseId].str(); + else + encodedPhraseCollection = m_phraseDictionary.m_targetPhrasesMapped[sourcePhraseId].str(); + + BitWrapper<> encodedBitStream(encodedPhraseCollection); + if(m_coding == PREnc && bitsLeft) + encodedBitStream.SeekFromEnd(bitsLeft); + + // Decompress and decode target phrase collection + TargetPhraseVectorPtr decodedPhraseColl = + DecodeCollection(mgr, tpv, encodedBitStream, sourcePhrase, topLevel, eval); + + return decodedPhraseColl; + } else + return TargetPhraseVectorPtr(); +} + +TargetPhraseVectorPtr PhraseDecoder::DecodeCollection( + const ManagerBase &mgr, + TargetPhraseVectorPtr tpv, + BitWrapper<> &encodedBitStream, + const Phrase<Word> &sourcePhrase, + bool topLevel, + bool eval) +{ + const System &system = mgr.system; + FactorCollection &vocab = system.GetVocab(); + + bool extending = tpv->size(); + size_t bitsLeft = encodedBitStream.TellFromEnd(); + + std::vector<int> sourceWords; + if(m_coding == REnc) { + for(size_t i = 0; i < sourcePhrase.GetSize(); i++) { + std::string sourceWord + = sourcePhrase[i].GetString(*m_input); + unsigned idx = GetSourceSymbolId(sourceWord); + sourceWords.push_back(idx); + } + } + + unsigned phraseStopSymbol = 0; + AlignPoint alignStopSymbol(-1, -1); + + std::vector<float> scores; + std::set<AlignPointSizeT> alignment; + + enum DecodeState { New, Symbol, Score, Alignment, Add } state = New; + + size_t srcSize = sourcePhrase.GetSize(); + + TPCompact* targetPhrase = NULL; + while(encodedBitStream.TellFromEnd()) { + + if(state == New) { + // Creating new TargetPhrase on the heap + tpv->push_back(TPCompact()); + targetPhrase = &tpv->back(); + + alignment.clear(); + scores.clear(); + + state = Symbol; + } + + if(state == Symbol) { + unsigned symbol = m_symbolTree->Read(encodedBitStream); + if(symbol == phraseStopSymbol) { + state = Score; + } else { + if(m_coding == REnc) { + std::string wordString; + size_t type = GetREncType(symbol); + + if(type == 1) { + unsigned decodedSymbol = DecodeREncSymbol1(symbol); + wordString = GetTargetSymbol(decodedSymbol); + } else if (type == 2) { + size_t rank = DecodeREncSymbol2Rank(symbol); + size_t srcPos = DecodeREncSymbol2Position(symbol); + + if(srcPos >= sourceWords.size()) + return TargetPhraseVectorPtr(); + + wordString = GetTargetSymbol(GetTranslation(sourceWords[srcPos], rank)); + if(m_phraseDictionary.m_useAlignmentInfo) { + size_t trgPos = targetPhrase->words.size(); + alignment.insert(AlignPoint(srcPos, trgPos)); + } + } else if(type == 3) { + size_t rank = DecodeREncSymbol3(symbol); + size_t srcPos = targetPhrase->words.size(); + + if(srcPos >= sourceWords.size()) + return TargetPhraseVectorPtr(); + + wordString = GetTargetSymbol(GetTranslation(sourceWords[srcPos], rank)); + if(m_phraseDictionary.m_useAlignmentInfo) { + size_t trgPos = srcPos; + alignment.insert(AlignPoint(srcPos, trgPos)); + } + } + + Word word; + word.CreateFromString(vocab, system, wordString); + targetPhrase->words.push_back(word); + } else if(m_coding == PREnc) { + // if the symbol is just a word + if(GetPREncType(symbol) == 1) { + unsigned decodedSymbol = DecodePREncSymbol1(symbol); + + Word word; + word.CreateFromString(vocab, system, GetTargetSymbol(decodedSymbol)); + targetPhrase->words.push_back(word); + } + // if the symbol is a subphrase pointer + else { + int left = DecodePREncSymbol2Left(symbol); + int right = DecodePREncSymbol2Right(symbol); + unsigned rank = DecodePREncSymbol2Rank(symbol); + + int srcStart = left + targetPhrase->words.size(); + int srcEnd = srcSize - right - 1; + + // false positive consistency check + if(0 > srcStart || srcStart > srcEnd || unsigned(srcEnd) >= srcSize) + return TargetPhraseVectorPtr(); + + // false positive consistency check + if(m_maxRank && rank > m_maxRank) + return TargetPhraseVectorPtr(); + + // set subphrase by default to itself + TargetPhraseVectorPtr subTpv = tpv; + + // if range smaller than source phrase retrieve subphrase + if(unsigned(srcEnd - srcStart + 1) != srcSize) { + SubPhrase<Word> subPhrase = sourcePhrase.GetSubPhrase(srcStart, srcEnd - srcStart + 1); + subTpv = CreateTargetPhraseCollection(mgr, subPhrase, false); + } else { + // false positive consistency check + if(rank >= tpv->size()-1) + return TargetPhraseVectorPtr(); + } + + // false positive consistency check + if(subTpv != NULL && rank < subTpv->size()) { + // insert the subphrase into the main target phrase + TPCompact& subTp = subTpv->at(rank); + if(m_phraseDictionary.m_useAlignmentInfo) { + // reconstruct the alignment data based on the alignment of the subphrase + for(std::set<AlignPointSizeT>::const_iterator it = subTp.alignment.begin(); + it != subTp.alignment.end(); it++) { + alignment.insert(AlignPointSizeT(srcStart + it->first, + targetPhrase->words.size() + it->second)); + } + } + + std::copy(subTp.words.begin(), subTp.words.end(), std::back_inserter(targetPhrase->words)); + } else + return TargetPhraseVectorPtr(); + } + } else { + Word word; + word.CreateFromString(vocab, system, GetTargetSymbol(symbol)); + targetPhrase->words.push_back(word); + } + } + } else if(state == Score) { + size_t idx = m_multipleScoreTrees ? scores.size() : 0; + float score = m_scoreTrees[idx]->Read(encodedBitStream); + scores.push_back(score); + + if(scores.size() == m_numScoreComponent) { + targetPhrase->scores = scores; + + if(m_containsAlignmentInfo) + state = Alignment; + else + state = Add; + } + } else if(state == Alignment) { + AlignPoint alignPoint = m_alignTree->Read(encodedBitStream); + if(alignPoint == alignStopSymbol) { + state = Add; + } else { + if(m_phraseDictionary.m_useAlignmentInfo) + alignment.insert(AlignPointSizeT(alignPoint)); + } + } + + if(state == Add) { + if(m_phraseDictionary.m_useAlignmentInfo) { + size_t sourceSize = sourcePhrase.GetSize(); + size_t targetSize = targetPhrase->words.size(); + for(std::set<AlignPointSizeT>::iterator it = alignment.begin(); it != alignment.end(); it++) { + if(it->first >= sourceSize || it->second >= targetSize) + return TargetPhraseVectorPtr(); + } + targetPhrase->alignment = alignment; + } + + if(m_coding == PREnc) { + if(!m_maxRank || tpv->size() <= m_maxRank) + bitsLeft = encodedBitStream.TellFromEnd(); + + if(!topLevel && m_maxRank && tpv->size() >= m_maxRank) + break; + } + + if(encodedBitStream.TellFromEnd() <= 8) + break; + + state = New; + } + } + + if(m_coding == PREnc && !extending) { + bitsLeft = bitsLeft > 8 ? bitsLeft : 0; + m_decodingCache.Cache(sourcePhrase, tpv, bitsLeft, m_maxRank); + } + + return tpv; +} + +void PhraseDecoder::PruneCache() +{ + m_decodingCache.Prune(); +} + +} diff --git a/moses2/TranslationModel/CompactPT/PhraseDecoder.h b/moses2/TranslationModel/CompactPT/PhraseDecoder.h new file mode 100644 index 000000000..79faa38a6 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/PhraseDecoder.h @@ -0,0 +1,142 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2006 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#pragma once + +#include <sstream> +#include <vector> +#include <boost/unordered_map.hpp> +#include <boost/unordered_set.hpp> +#include <string> +#include <iterator> +#include <algorithm> +#include <sys/stat.h> + +#include "PhraseTableCompact.h" +#include "StringVector.h" +#include "CanonicalHuffman.h" +#include "TargetPhraseCollectionCache.h" + +#include "../../Phrase.h" +#include "../../ManagerBase.h" + +namespace Moses2 +{ + +class PhraseTableCompact; + +class PhraseDecoder +{ +protected: + + friend class PhraseTableCompact; + + typedef std::pair<unsigned char, unsigned char> AlignPoint; + typedef std::pair<unsigned, unsigned> SrcTrg; + + enum Coding { None, REnc, PREnc } m_coding; + + size_t m_numScoreComponent; + bool m_containsAlignmentInfo; + size_t m_maxRank; + size_t m_maxPhraseLength; + + boost::unordered_map<std::string, unsigned> m_sourceSymbolsMap; + StringVector<unsigned char, unsigned, std::allocator> m_sourceSymbols; + StringVector<unsigned char, unsigned, std::allocator> m_targetSymbols; + + std::vector<size_t> m_lexicalTableIndex; + std::vector<SrcTrg> m_lexicalTable; + + CanonicalHuffman<unsigned>* m_symbolTree; + + bool m_multipleScoreTrees; + std::vector<CanonicalHuffman<float>*> m_scoreTrees; + + CanonicalHuffman<AlignPoint>* m_alignTree; + + TargetPhraseCollectionCache m_decodingCache; + + PhraseTableCompact& m_phraseDictionary; + + // *********************************************** + + const std::vector<FactorType>* m_input; + const std::vector<FactorType>* m_output; + + std::string m_separator; + + // *********************************************** + + unsigned GetSourceSymbolId(std::string& s); + std::string GetTargetSymbol(unsigned id) const; + + size_t GetREncType(unsigned encodedSymbol); + size_t GetPREncType(unsigned encodedSymbol); + + unsigned GetTranslation(unsigned srcIdx, size_t rank); + + size_t GetMaxSourcePhraseLength(); + + unsigned DecodeREncSymbol1(unsigned encodedSymbol); + unsigned DecodeREncSymbol2Rank(unsigned encodedSymbol); + unsigned DecodeREncSymbol2Position(unsigned encodedSymbol); + unsigned DecodeREncSymbol3(unsigned encodedSymbol); + + unsigned DecodePREncSymbol1(unsigned encodedSymbol); + int DecodePREncSymbol2Left(unsigned encodedSymbol); + int DecodePREncSymbol2Right(unsigned encodedSymbol); + unsigned DecodePREncSymbol2Rank(unsigned encodedSymbol); + + std::string MakeSourceKey(std::string &); + +public: + + PhraseDecoder( + PhraseTableCompact &phraseDictionary, + const std::vector<FactorType>* input, + const std::vector<FactorType>* output, + size_t numScoreComponent + ); + + ~PhraseDecoder(); + + size_t Load(std::FILE* in); + + TargetPhraseVectorPtr CreateTargetPhraseCollection( + const ManagerBase &mgr, + const Phrase<Word> &sourcePhrase, + bool topLevel = false, + bool eval = true); + + TargetPhraseVectorPtr DecodeCollection( + const ManagerBase &mgr, + TargetPhraseVectorPtr tpv, + BitWrapper<> &encodedBitStream, + const Phrase<Word> &sourcePhrase, + bool topLevel, + bool eval); + + void PruneCache(); +}; + +} + diff --git a/moses2/TranslationModel/CompactPT/PhraseTableCompact.cpp b/moses2/TranslationModel/CompactPT/PhraseTableCompact.cpp new file mode 100644 index 000000000..49244df1b --- /dev/null +++ b/moses2/TranslationModel/CompactPT/PhraseTableCompact.cpp @@ -0,0 +1,222 @@ +#include <boost/algorithm/string/predicate.hpp> +#include <boost/thread/tss.hpp> +#include "PhraseTableCompact.h" +#include "PhraseDecoder.h" +#include "../../PhraseBased/InputPath.h" +#include "../../PhraseBased/Manager.h" +#include "../../PhraseBased/TargetPhrases.h" +#include "../../PhraseBased/TargetPhraseImpl.h" +#include "../../PhraseBased/Sentence.h" + +using namespace std; +using namespace boost::algorithm; + +namespace Moses2 +{ +bool PhraseTableCompact::s_inMemoryByDefault = false; + +PhraseTableCompact::PhraseTableCompact(size_t startInd, const std::string &line) +:PhraseTable(startInd, line) +,m_inMemory(s_inMemoryByDefault) +,m_useAlignmentInfo(true) +,m_hash(10, 16) +,m_phraseDecoder(0) +{ + ReadParameters(); +} + +PhraseTableCompact::~PhraseTableCompact() +{ + +} + +void PhraseTableCompact::Load(System &system) +{ + std::string tFilePath = m_path; + + std::string suffix = ".minphr"; + if (!ends_with(tFilePath, suffix)) tFilePath += suffix; + if (!FileExists(tFilePath)) + throw runtime_error("Error: File " + tFilePath + " does not exist."); + + m_phraseDecoder + = new PhraseDecoder(*this, &m_input, &m_output, GetNumScores()); + + std::FILE* pFile = std::fopen(tFilePath.c_str() , "r"); + + size_t indexSize; + //if(m_inMemory) + // Load source phrase index into memory + indexSize = m_hash.Load(pFile); + // else + // Keep source phrase index on disk + //indexSize = m_hash.LoadIndex(pFile); + + size_t coderSize = m_phraseDecoder->Load(pFile); + + size_t phraseSize; + if(m_inMemory) { + // Load target phrase collections into memory + phraseSize = m_targetPhrasesMemory.load(pFile, false); + } + else { + // Keep target phrase collections on disk + phraseSize = m_targetPhrasesMapped.load(pFile, true); + } + + UTIL_THROW_IF2(indexSize == 0 || coderSize == 0 || phraseSize == 0, + "Not successfully loaded"); +} + +void PhraseTableCompact::SetParameter(const std::string& key, const std::string& value) +{ + if (key == "blah") { + + } + else { + PhraseTable::SetParameter(key, value); + } +} + +void PhraseTableCompact::CleanUpAfterSentenceProcessing() const +{ + //if(!m_sentenceCache.get()) + // m_sentenceCache.reset(new PhraseCache()); + + m_phraseDecoder->PruneCache(); + //m_sentenceCache->clear(); +} + + +// pb +void PhraseTableCompact::Lookup(const Manager &mgr, InputPathsBase &inputPaths) const +{ + size_t inputSize = static_cast<const Sentence&>(mgr.GetInput()).GetSize(); + InputPaths &inputPathsCast = static_cast<InputPaths&>(inputPaths); + + for (size_t i = 0; i < inputSize; ++i) { + for (size_t startPos = 0; startPos < inputSize; ++startPos) { + size_t endPos = startPos + i; + if (endPos >= inputSize) { + break; + } + InputPath *path = inputPathsCast.GetMatrix().GetValue(startPos, i); + //cerr << "path=" << path->Debug(mgr.system) << endl; + TargetPhrases *tps = Lookup(mgr, mgr.GetPool(), *path); + path->AddTargetPhrases(*this, tps); + } + } +} + +TargetPhrases *PhraseTableCompact::Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const +{ + TargetPhrases *ret = NULL; + + const Phrase<Word> &sourcePhrase = inputPath.subPhrase; + //cerr << "sourcePhrase=" << sourcePhrase.Debug(mgr.system) << endl; + + // There is no souch source phrase if source phrase is longer than longest + // observed source phrase during compilation + if(sourcePhrase.GetSize() > m_phraseDecoder->GetMaxSourcePhraseLength()) + return ret; + + // Retrieve target phrase collection from phrase table + TargetPhraseVectorPtr decodedPhraseColl + = m_phraseDecoder->CreateTargetPhraseCollection(mgr, sourcePhrase, true, true); + + if(decodedPhraseColl != NULL && decodedPhraseColl->size()) { + TargetPhraseVectorPtr tpv(new TargetPhraseVector(*decodedPhraseColl)); + //TargetPhraseCollection::shared_ptr phraseColl(new TargetPhraseCollection); + ret = new (pool.Allocate<TargetPhrases>()) TargetPhrases(pool, decodedPhraseColl->size()); + + for (size_t i = 0; i < decodedPhraseColl->size(); ++i) { + const TPCompact &tpCompact = decodedPhraseColl->at(i); + const TargetPhraseImpl *tp = CreateTargetPhrase(mgr, tpCompact, sourcePhrase); + + ret->AddTargetPhrase(*tp); + } + + ret->SortAndPrune(m_tableLimit); + mgr.system.featureFunctions.EvaluateAfterTablePruning(pool, *ret, sourcePhrase); + + //cerr << "RET2=" << ret->Debug(mgr.system) << endl; + /* + // Cache phrase pair for clean-up or retrieval with PREnc + const_cast<PhraseDictionaryCompact*>(this)->CacheForCleanup(phraseColl); + + return phraseColl; + */ + } + + return ret; + +} + +const TargetPhraseImpl *PhraseTableCompact::CreateTargetPhrase( + const Manager &mgr, + const TPCompact &tpCompact, + const Phrase<Word> &sourcePhrase) const +{ + MemPool &pool = mgr.GetPool(); + + size_t size = tpCompact.words.size(); + TargetPhraseImpl *ret = new TargetPhraseImpl(pool, *this, mgr.system, size); + + // words + for (size_t i = 0; i < size; ++i) { + const Word &compactWord = tpCompact.words[i]; + Word &tpWord = (*ret)[i]; + tpWord = compactWord; + } + + // scores + Scores &scores = ret->GetScores(); + scores.Assign(mgr.system, *this, tpCompact.scores); + + // align + ret->SetAlignTerm(tpCompact.alignment); + + // score + mgr.system.featureFunctions.EvaluateInIsolation(pool, mgr.system, sourcePhrase, *ret); + + // Cache phrase pair for clean-up or retrieval with PREnc + //const_cast<PhraseDictionaryCompact*>(this)->CacheForCleanup(phraseColl); + + //cerr << "ret=" << ret->Debug(mgr.system) << endl; + return ret; +} + + +// scfg +void PhraseTableCompact::InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const +{ + UTIL_THROW2("Not implemented"); +} + +void PhraseTableCompact::Lookup( + MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const +{ + UTIL_THROW2("Not implemented"); +} + +void PhraseTableCompact::LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const +{ + UTIL_THROW2("Not implemented"); +} + +} diff --git a/moses2/TranslationModel/CompactPT/PhraseTableCompact.h b/moses2/TranslationModel/CompactPT/PhraseTableCompact.h new file mode 100644 index 000000000..84ea7e4b2 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/PhraseTableCompact.h @@ -0,0 +1,68 @@ +#pragma once +#include "../PhraseTable.h" +#include "BlockHashIndex.h" + +namespace Moses2 +{ +class PhraseDecoder; +class TPCompact; + +class PhraseTableCompact: public PhraseTable +{ +public: + PhraseTableCompact(size_t startInd, const std::string &line); + virtual ~PhraseTableCompact(); + void Load(System &system); + virtual void SetParameter(const std::string& key, const std::string& value); + + virtual void CleanUpAfterSentenceProcessing() const; + + virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const; + + // scfg + virtual void InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; + + virtual void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const; + + virtual void Lookup( + MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; + +protected: + static bool s_inMemoryByDefault; + bool m_inMemory; + bool m_useAlignmentInfo; + + BlockHashIndex m_hash; + + StringVector<unsigned char, size_t, MmapAllocator> m_targetPhrasesMapped; + StringVector<unsigned char, size_t, std::allocator> m_targetPhrasesMemory; + + friend class PhraseDecoder; + PhraseDecoder* m_phraseDecoder; + + const TargetPhraseImpl *CreateTargetPhrase( + const Manager &mgr, + const TPCompact &tpCompact, + const Phrase<Word> &sourcePhrase) const; + + // SCFG + virtual void LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; + +}; + +} diff --git a/moses2/TranslationModel/CompactPT/StringVector.h b/moses2/TranslationModel/CompactPT/StringVector.h new file mode 100644 index 000000000..87d6388bf --- /dev/null +++ b/moses2/TranslationModel/CompactPT/StringVector.h @@ -0,0 +1,662 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_StringVector_h +#define moses_StringVector_h + +#include <vector> +#include <algorithm> +#include <string> +#include <iterator> +#include <cstdio> +#include <cassert> + +#include <boost/iterator/iterator_facade.hpp> + +#include "ThrowingFwrite.h" +#include "MonotonicVector.h" +#include "MmapAllocator.h" + +namespace Moses2 +{ + +// ********** ValueIteratorRange ********** + +template<typename ValueIteratorT> +class ValueIteratorRange +{ +private: + ValueIteratorT m_begin; + ValueIteratorT m_end; + +public: + ValueIteratorRange(ValueIteratorT begin, ValueIteratorT end); + + const ValueIteratorT& begin() const; + const ValueIteratorT& end() const; + const std::string str() const; + operator const std::string() + { + return str(); + } + + size_t size() + { + return std::distance(m_begin, m_end); + } + + template<typename StringT> + bool operator==(const StringT& o) const; + bool operator==(const char* c) const; + + template<typename StringT> + bool operator<(const StringT& o) const; + bool operator<(const char* c) const; +}; + +// ********** StringVector ********** + +template<typename ValueT = unsigned char, typename PosT = unsigned int, + template<typename > class Allocator = std::allocator> +class StringVector +{ +protected: + bool m_sorted; + bool m_memoryMapped; + + std::vector<ValueT, Allocator<ValueT> >* m_charArray; + MonotonicVector<PosT, unsigned int, 32> m_positions; + + virtual const ValueT* value_ptr(PosT i) const; + +public: + //typedef ValueIteratorRange<typename std::vector<ValueT, Allocator<ValueT> >::const_iterator> range; + typedef ValueIteratorRange<const ValueT *> range; + + // ********** RangeIterator ********** + + class RangeIterator: public boost::iterator_facade<RangeIterator, range, + std::random_access_iterator_tag, range, PosT> + { + + private: + PosT m_index; + StringVector<ValueT, PosT, Allocator>* m_container; + + public: + RangeIterator(); + RangeIterator(StringVector<ValueT, PosT, Allocator> &sv, PosT index = 0); + + PosT get_index(); + + private: + friend class boost::iterator_core_access; + + range dereference() const; + bool equal(RangeIterator const& other) const; + void increment(); + void decrement(); + void advance(PosT n); + + PosT distance_to(RangeIterator const& other) const; + }; + + // ********** StringIterator ********** + + class StringIterator: public boost::iterator_facade<StringIterator, + std::string, std::random_access_iterator_tag, const std::string, PosT> + { + + private: + PosT m_index; + StringVector<ValueT, PosT, Allocator>* m_container; + + public: + StringIterator(); + StringIterator(StringVector<ValueT, PosT, Allocator> &sv, PosT index = 0); + + PosT get_index(); + + private: + friend class boost::iterator_core_access; + + const std::string dereference() const; + bool equal(StringIterator const& other) const; + void increment(); + void decrement(); + void advance(PosT n); + PosT distance_to(StringIterator const& other) const; + }; + + typedef RangeIterator iterator; + typedef StringIterator string_iterator; + + StringVector(bool allocate = false); + StringVector(Allocator<ValueT>& alloc); + + virtual ~StringVector() + { + delete m_charArray; + } + + void swap(StringVector<ValueT, PosT, Allocator> &c) + { + m_positions.commit(); + m_positions.swap(c.m_positions); + m_charArray->swap(*c.m_charArray); + + bool temp = m_sorted; + m_sorted = c.m_sorted; + c.m_sorted = temp; + } + + bool is_sorted() const; + PosT size() const; + virtual PosT size2() const; + + template<class Iterator> Iterator begin() const; + template<class Iterator> Iterator end() const; + + iterator begin() const; + iterator end() const; + + PosT length(PosT i) const; + //typename std::vector<ValueT, Allocator<ValueT> >::const_iterator begin(PosT i) const; + //typename std::vector<ValueT, Allocator<ValueT> >::const_iterator end(PosT i) const; + const ValueT* begin(PosT i) const; + const ValueT* end(PosT i) const; + + void clear() + { + m_charArray->clear(); + m_sorted = true; + m_positions = MonotonicVector<PosT, unsigned int, 32>(); + } + + range at(PosT i) const; + range operator[](PosT i) const; + range back() const; + + template<typename StringT> + void push_back(StringT s); + void push_back(const char* c); + + template<typename StringT> + PosT find(StringT &s) const; + PosT find(const char* c) const; + + virtual size_t load(std::FILE* in, bool memoryMapped = false) + { + size_t size = 0; + m_memoryMapped = memoryMapped; + + size += std::fread(&m_sorted, sizeof(bool), 1, in) * sizeof(bool); + size += m_positions.load(in, false); + + size += loadCharArray(m_charArray, in, m_memoryMapped); + return size; + } + + size_t loadCharArray(std::vector<ValueT, std::allocator<ValueT> >*& c, + std::FILE* in, bool map = false) + { + // Can only be read into memory. Mapping not possible with std:allocator. + assert(map == false); + + size_t byteSize = 0; + + size_t valSize; + byteSize += std::fread(&valSize, sizeof(size_t), 1, in) * sizeof(size_t); + + c = new std::vector<ValueT, std::allocator<ValueT> >(valSize, 0); + byteSize += std::fread(&(*c)[0], sizeof(ValueT), valSize, in) + * sizeof(ValueT); + + return byteSize; + } + + size_t loadCharArray(std::vector<ValueT, MmapAllocator<ValueT> >*& c, + std::FILE* in, bool map = false) + { + size_t byteSize = 0; + + size_t valSize; + byteSize += std::fread(&valSize, sizeof(size_t), 1, in) * sizeof(size_t); + + if (map == false) { + // Read data into temporary file (default constructor of MmapAllocator) + // and map memory onto temporary file. Can be resized. + c = new std::vector<ValueT, MmapAllocator<ValueT> >(valSize, 0); + byteSize += std::fread(&(*c)[0], sizeof(ValueT), valSize, in) + * sizeof(ValueT); + } + else { + // Map it directly on specified region of file "in" starting at valPos + // with length valSize * sizeof(ValueT). Mapped region cannot be resized. + + size_t valPos = std::ftell(in); + Allocator<ValueT> alloc(in, valPos); + c = new std::vector<ValueT, Allocator<ValueT> >(alloc); + c->resize(valSize, 0); + + byteSize += valSize * sizeof(ValueT); + } + + return byteSize; + } + + size_t load(std::string filename, bool memoryMapped = false) + { + std::FILE* pFile = fopen(filename.c_str(), "r"); + size_t byteSize = load(pFile, memoryMapped); + fclose(pFile); + return byteSize; + } + + size_t save(std::FILE* out) + { + size_t byteSize = 0; + byteSize += ThrowingFwrite(&m_sorted, sizeof(bool), 1, out) * sizeof(bool); + + byteSize += m_positions.save(out); + + size_t valSize = size2(); + byteSize += ThrowingFwrite(&valSize, sizeof(size_t), 1, out) + * sizeof(size_t); + byteSize += ThrowingFwrite(&(*m_charArray)[0], sizeof(ValueT), valSize, out) + * sizeof(ValueT); + + return byteSize; + } + + size_t save(std::string filename) + { + std::FILE* pFile = fopen(filename.c_str(), "w"); + size_t byteSize = save(pFile); + fclose(pFile); + return byteSize; + } + +}; + +// ********** Implementation ********** + +// ValueIteratorRange + +template<typename ValueIteratorT> +ValueIteratorRange<ValueIteratorT>::ValueIteratorRange(ValueIteratorT begin, + ValueIteratorT end) : + m_begin(begin), m_end(end) +{ +} + +template<typename ValueIteratorT> +const ValueIteratorT& ValueIteratorRange<ValueIteratorT>::begin() const +{ + return m_begin; +} + +template<typename ValueIteratorT> +const ValueIteratorT& ValueIteratorRange<ValueIteratorT>::end() const +{ + return m_end; +} + +template<typename ValueIteratorT> +const std::string ValueIteratorRange<ValueIteratorT>::str() const +{ + std::string dummy; + for (ValueIteratorT it = m_begin; it != m_end; it++) + dummy.push_back(*it); + return dummy; +} + +template<typename ValueIteratorT> +template<typename StringT> +bool ValueIteratorRange<ValueIteratorT>::operator==(const StringT& o) const +{ + if (std::distance(m_begin, m_end) == std::distance(o.begin(), o.end())) return std::equal( + m_begin, m_end, o.begin()); + else return false; +} + +template<typename ValueIteratorT> +bool ValueIteratorRange<ValueIteratorT>::operator==(const char* c) const +{ + return *this == std::string(c); +} + +template<typename ValueIteratorT> +template<typename StringT> +bool ValueIteratorRange<ValueIteratorT>::operator<(const StringT &s2) const +{ + return std::lexicographical_compare(m_begin, m_end, s2.begin(), s2.end(), + std::less<typename std::iterator_traits<ValueIteratorT>::value_type>()); +} + +template<typename ValueIteratorT> +bool ValueIteratorRange<ValueIteratorT>::operator<(const char* c) const +{ + return *this < std::string(c); +} + +template<typename StringT, typename ValueIteratorT> +bool operator<(const StringT &s1, const ValueIteratorRange<ValueIteratorT> &s2) +{ + return std::lexicographical_compare(s1.begin(), s1.end(), s2.begin(), + s2.end(), + std::less<typename std::iterator_traits<ValueIteratorT>::value_type>()); +} + +template<typename ValueIteratorT> +bool operator<(const char* c, const ValueIteratorRange<ValueIteratorT> &s2) +{ + size_t len = std::char_traits<char>::length(c); + return std::lexicographical_compare(c, c + len, s2.begin(), s2.end(), + std::less<typename std::iterator_traits<ValueIteratorT>::value_type>()); +} + +template<typename OStream, typename ValueIteratorT> +OStream& operator<<(OStream &os, ValueIteratorRange<ValueIteratorT> cr) +{ + ValueIteratorT it = cr.begin(); + while (it != cr.end()) + os << *(it++); + return os; +} + +// StringVector + +template<typename ValueT, typename PosT, template<typename > class Allocator> +StringVector<ValueT, PosT, Allocator>::StringVector(bool allocate) : + m_sorted(true), m_memoryMapped(false), m_charArray( + allocate ? new std::vector<ValueT, Allocator<ValueT> >() : 0) +{ +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +StringVector<ValueT, PosT, Allocator>::StringVector(Allocator<ValueT> &alloc) : + m_sorted(true), m_memoryMapped(false), m_charArray( + new std::vector<ValueT, Allocator<ValueT> >(alloc)) +{ +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +template<typename StringT> +void StringVector<ValueT, PosT, Allocator>::push_back(StringT s) +{ + if (is_sorted() && size() && !(back() < s)) m_sorted = false; + + m_positions.push_back(size2()); + std::copy(s.begin(), s.end(), std::back_inserter(*m_charArray)); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +void StringVector<ValueT, PosT, Allocator>::push_back(const char* c) +{ + std::string dummy(c); + push_back(dummy); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +template<typename Iterator> +Iterator StringVector<ValueT, PosT, Allocator>::begin() const +{ + return Iterator(const_cast<StringVector<ValueT, PosT, Allocator>&>(*this), 0); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +template<typename Iterator> +Iterator StringVector<ValueT, PosT, Allocator>::end() const +{ + return Iterator(const_cast<StringVector<ValueT, PosT, Allocator>&>(*this), + size()); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +typename StringVector<ValueT, PosT, Allocator>::iterator StringVector<ValueT, + PosT, Allocator>::begin() const +{ + return begin<iterator>(); +} +; + +template<typename ValueT, typename PosT, template<typename > class Allocator> +typename StringVector<ValueT, PosT, Allocator>::iterator StringVector<ValueT, + PosT, Allocator>::end() const +{ + return end<iterator>(); +} +; + +template<typename ValueT, typename PosT, template<typename > class Allocator> +bool StringVector<ValueT, PosT, Allocator>::is_sorted() const +{ + return m_sorted; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +PosT StringVector<ValueT, PosT, Allocator>::size() const +{ + return m_positions.size(); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +PosT StringVector<ValueT, PosT, Allocator>::size2() const +{ + return m_charArray->size(); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +typename StringVector<ValueT, PosT, Allocator>::range StringVector<ValueT, PosT, + Allocator>::at(PosT i) const +{ + return range(begin(i), end(i)); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +typename StringVector<ValueT, PosT, Allocator>::range StringVector<ValueT, PosT, + Allocator>::operator[](PosT i) const +{ + return at(i); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +typename StringVector<ValueT, PosT, Allocator>::range StringVector<ValueT, PosT, + Allocator>::back() const +{ + return at(size() - 1); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +PosT StringVector<ValueT, PosT, Allocator>::length(PosT i) const +{ + if (i + 1 < size()) return m_positions[i + 1] - m_positions[i]; + else return size2() - m_positions[i]; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +const ValueT* StringVector<ValueT, PosT, Allocator>::value_ptr(PosT i) const +{ + return &(*m_charArray)[m_positions[i]]; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +//typename std::vector<ValueT, Allocator<ValueT> >::const_iterator StringVector<ValueT, PosT, Allocator>::begin(PosT i) const +const ValueT* StringVector<ValueT, PosT, Allocator>::begin(PosT i) const +{ + //return typename std::vector<ValueT, Allocator<ValueT> >::const_iterator(value_ptr(i)); + return value_ptr(i); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +//typename std::vector<ValueT, Allocator<ValueT> >::const_iterator StringVector<ValueT, PosT, Allocator>::end(PosT i) const +const ValueT* StringVector<ValueT, PosT, Allocator>::end(PosT i) const +{ + //return typename std::vector<ValueT, Allocator<ValueT> >::const_iterator(value_ptr(i) + length(i)); + return value_ptr(i) + length(i); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +template<typename StringT> +PosT StringVector<ValueT, PosT, Allocator>::find(StringT &s) const +{ + if (m_sorted) return std::distance(begin(), + std::lower_bound(begin(), end(), s)); + return std::distance(begin(), std::find(begin(), end(), s)); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +PosT StringVector<ValueT, PosT, Allocator>::find(const char* c) const +{ + std::string s(c); + return find(s); +} + +// RangeIterator + +template<typename ValueT, typename PosT, template<typename > class Allocator> +StringVector<ValueT, PosT, Allocator>::RangeIterator::RangeIterator() : + m_index(0), m_container(0) +{ +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +StringVector<ValueT, PosT, Allocator>::RangeIterator::RangeIterator( + StringVector<ValueT, PosT, Allocator> &sv, PosT index) : + m_index(index), m_container(&sv) +{ +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +PosT StringVector<ValueT, PosT, Allocator>::RangeIterator::get_index() +{ + return m_index; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +typename StringVector<ValueT, PosT, Allocator>::range StringVector<ValueT, PosT, + Allocator>::RangeIterator::dereference() const +{ + return typename StringVector<ValueT, PosT, Allocator>::range( + m_container->begin(m_index), m_container->end(m_index)); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +bool StringVector<ValueT, PosT, Allocator>::RangeIterator::equal( + StringVector<ValueT, PosT, Allocator>::RangeIterator const& other) const +{ + return m_index == other.m_index && m_container == other.m_container; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +void StringVector<ValueT, PosT, Allocator>::RangeIterator::increment() +{ + m_index++; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +void StringVector<ValueT, PosT, Allocator>::RangeIterator::decrement() +{ + m_index--; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +void StringVector<ValueT, PosT, Allocator>::RangeIterator::advance(PosT n) +{ + m_index += n; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +PosT StringVector<ValueT, PosT, Allocator>::RangeIterator::distance_to( + StringVector<ValueT, PosT, Allocator>::RangeIterator const& other) const +{ + return other.m_index - m_index; +} + +// StringIterator + +template<typename ValueT, typename PosT, template<typename > class Allocator> +StringVector<ValueT, PosT, Allocator>::StringIterator::StringIterator() : + m_index(0), m_container(0) +{ +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +StringVector<ValueT, PosT, Allocator>::StringIterator::StringIterator( + StringVector<ValueT, PosT, Allocator> &sv, PosT index) : + m_index(index), m_container(&sv) +{ +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +PosT StringVector<ValueT, PosT, Allocator>::StringIterator::get_index() +{ + return m_index; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +const std::string StringVector<ValueT, PosT, Allocator>::StringIterator::dereference() const +{ + return StringVector<ValueT, PosT, Allocator>::range( + m_container->begin(m_index), m_container->end(m_index)).str(); +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +bool StringVector<ValueT, PosT, Allocator>::StringIterator::equal( + StringVector<ValueT, PosT, Allocator>::StringIterator const& other) const +{ + return m_index == other.m_index && m_container == other.m_container; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +void StringVector<ValueT, PosT, Allocator>::StringIterator::increment() +{ + m_index++; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +void StringVector<ValueT, PosT, Allocator>::StringIterator::decrement() +{ + m_index--; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +void StringVector<ValueT, PosT, Allocator>::StringIterator::advance(PosT n) +{ + m_index += n; +} + +template<typename ValueT, typename PosT, template<typename > class Allocator> +PosT StringVector<ValueT, PosT, Allocator>::StringIterator::distance_to( + StringVector<ValueT, PosT, Allocator>::StringIterator const& other) const +{ + return other.m_index - m_index; +} + +// ********** Some typedefs ********** + +typedef StringVector<unsigned char, unsigned int> MediumStringVector; +typedef StringVector<unsigned char, unsigned long> LongStringVector; + +} + +#endif diff --git a/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp b/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp new file mode 100644 index 000000000..07d0469e0 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.cpp @@ -0,0 +1,39 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2006 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#include "TargetPhraseCollectionCache.h" + +namespace Moses2 +{ + +boost::thread_specific_ptr<TargetPhraseCollectionCache::CacheMap> +TargetPhraseCollectionCache::m_phraseCache; + +PhraseCompact::PhraseCompact(const Phrase<Word> ©) +{ + for (size_t i = 0; i < copy.GetSize(); ++i) { + const Word &word = copy[i]; + push_back(word); + } +} + +} + diff --git a/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.h b/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.h new file mode 100644 index 000000000..3a9e6f170 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/TargetPhraseCollectionCache.h @@ -0,0 +1,176 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** +Moses - factored phrase-based language decoder +Copyright (C) 2006 University of Edinburgh + +This library is free software; you can redistribute it and/or +modify it under the terms of the GNU Lesser General Public +License as published by the Free Software Foundation; either +version 2.1 of the License, or (at your option) any later version. + +This library is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +Lesser General Public License for more details. + +You should have received a copy of the GNU Lesser General Public +License along with this library; if not, write to the Free Software +Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA +***********************************************************************/ + +#pragma once + +#include <map> +#include <set> +#include <vector> + +#include <boost/thread/tss.hpp> +#include <boost/shared_ptr.hpp> + +#include "../../Word.h" +#include "../../Phrase.h" + +namespace Moses2 +{ +typedef std::pair<size_t, size_t> AlignPointSizeT; + +struct PhraseCompact : public std::vector<Word> +{ +public: + PhraseCompact(const Phrase<Word> ©); +}; + +struct TPCompact +{ + std::vector<Word> words; + std::set<AlignPointSizeT> alignment; + std::vector<float> scores; + +}; + +// Avoid using new due to locking +typedef std::vector<TPCompact> TargetPhraseVector; +typedef boost::shared_ptr<TargetPhraseVector> TargetPhraseVectorPtr; + +/** Implementation of Persistent Cache **/ +class TargetPhraseCollectionCache +{ +private: + size_t m_max; + float m_tolerance; + + struct LastUsed { + clock_t m_clock; + TargetPhraseVectorPtr m_tpv; + size_t m_bitsLeft; + + LastUsed() : m_clock(0), m_bitsLeft(0) {} + + LastUsed(clock_t clock, TargetPhraseVectorPtr tpv, size_t bitsLeft = 0) + : m_clock(clock), m_tpv(tpv), m_bitsLeft(bitsLeft) {} + }; + + typedef std::map<PhraseCompact, LastUsed> CacheMap; + static boost::thread_specific_ptr<CacheMap> m_phraseCache; + +public: + + typedef CacheMap::iterator iterator; + typedef CacheMap::const_iterator const_iterator; + + TargetPhraseCollectionCache(size_t max = 5000, float tolerance = 0.2) + : m_max(max), m_tolerance(tolerance) { + } + + iterator Begin() { + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + return m_phraseCache->begin(); + } + + const_iterator Begin() const { + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + return m_phraseCache->begin(); + } + + iterator End() { + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + return m_phraseCache->end(); + } + + const_iterator End() const { + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + return m_phraseCache->end(); + } + + /** retrieve translations for source phrase from persistent cache **/ + void Cache(const Phrase<Word> &sourcePhrase, TargetPhraseVectorPtr tpv, + size_t bitsLeft = 0, size_t maxRank = 0) { + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + // check if source phrase is already in cache + iterator it = m_phraseCache->find(sourcePhrase); + if(it != m_phraseCache->end()) + // if found, just update clock + it->second.m_clock = clock(); + else { + // else, add to cache + if(maxRank && tpv->size() > maxRank) { + TargetPhraseVectorPtr tpv_temp(new TargetPhraseVector()); + tpv_temp->resize(maxRank); + std::copy(tpv->begin(), tpv->begin() + maxRank, tpv_temp->begin()); + (*m_phraseCache)[sourcePhrase] = LastUsed(clock(), tpv_temp, bitsLeft); + } else + (*m_phraseCache)[sourcePhrase] = LastUsed(clock(), tpv, bitsLeft); + } + } + + std::pair<TargetPhraseVectorPtr, size_t> Retrieve(const Phrase<Word> &sourcePhrase) { + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + iterator it = m_phraseCache->find(sourcePhrase); + if(it != m_phraseCache->end()) { + LastUsed &lu = it->second; + lu.m_clock = clock(); + return std::make_pair(lu.m_tpv, lu.m_bitsLeft); + } else + return std::make_pair(TargetPhraseVectorPtr(), 0); + } + + // if cache full, reduce + void Prune() { + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + if(m_phraseCache->size() > m_max * (1 + m_tolerance)) { + typedef std::set<std::pair<clock_t, PhraseCompact > > Cands; + Cands cands; + for(CacheMap::iterator it = m_phraseCache->begin(); + it != m_phraseCache->end(); it++) { + LastUsed &lu = it->second; + cands.insert(std::make_pair(lu.m_clock, it->first)); + } + + for(Cands::iterator it = cands.begin(); it != cands.end(); it++) { + const PhraseCompact& p = it->second; + m_phraseCache->erase(p); + + if(m_phraseCache->size() < (m_max * (1 - m_tolerance))) + break; + } + } + } + + void CleanUp() { + if(!m_phraseCache.get()) + m_phraseCache.reset(new CacheMap()); + m_phraseCache->clear(); + } + +}; + +} + diff --git a/moses2/TranslationModel/CompactPT/ThrowingFwrite.cpp b/moses2/TranslationModel/CompactPT/ThrowingFwrite.cpp new file mode 100644 index 000000000..d9fec5013 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/ThrowingFwrite.cpp @@ -0,0 +1,30 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#include "ThrowingFwrite.h" + +size_t ThrowingFwrite(const void *ptr, size_t size, size_t count, FILE* stream) +{ + assert(size); + size_t returnValue = std::fwrite(ptr, size, count, stream); + UTIL_THROW_IF2(count != returnValue, "Short fwrite; requested size " << size); + return returnValue; +} diff --git a/moses2/TranslationModel/CompactPT/ThrowingFwrite.h b/moses2/TranslationModel/CompactPT/ThrowingFwrite.h new file mode 100644 index 000000000..2a0c71a27 --- /dev/null +++ b/moses2/TranslationModel/CompactPT/ThrowingFwrite.h @@ -0,0 +1,31 @@ +// $Id$ +// vim:tabstop=2 +/*********************************************************************** + Moses - factored phrase-based language decoder + Copyright (C) 2006 University of Edinburgh + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + ***********************************************************************/ + +#ifndef moses_ThrowingFwrite_h +#define moses_ThrowingFwrite_h + +#include <cassert> +#include <cstdio> +#include "util/exception.hh" + +size_t ThrowingFwrite(const void *ptr, size_t size, size_t count, FILE* stream); + +#endif diff --git a/moses2/TranslationModel/Memory/Node.h b/moses2/TranslationModel/Memory/Node.h new file mode 100644 index 000000000..97fa9618e --- /dev/null +++ b/moses2/TranslationModel/Memory/Node.h @@ -0,0 +1,138 @@ +/* + * Node.h + * + * Created on: 22 Apr 2016 + * Author: hieu + */ +#pragma once +#include <boost/unordered_map.hpp> +#include <boost/foreach.hpp> +#include "../../PhraseBased/TargetPhrases.h" +#include "../../System.h" +#include "../../Phrase.h" + +namespace Moses2 +{ +class System; + +namespace PtMem +{ + +template<class WORD, class SP, class TP, class TPS> +class Node +{ +public: + typedef boost::unordered_map<size_t, Node> Children; + + Node() + :m_targetPhrases(NULL) + ,m_unsortedTPS(NULL) + {} + + ~Node() + {} + + void AddRule(const std::vector<FactorType> &factors, SP &source, TP *target) + { + AddRule(factors, source, target, 0); + } + + TPS *Find(const std::vector<FactorType> &factors, const SP &source, size_t pos = 0) const + { + assert(source.GetSize()); + if (pos == source.GetSize()) { + return m_targetPhrases; + } + else { + const WORD &word = source[pos]; + //cerr << "word=" << word << endl; + typename Children::const_iterator iter = m_children.find(word.hash(factors)); + if (iter == m_children.end()) { + return NULL; + } + else { + const Node &child = iter->second; + return child.Find(factors, source, pos + 1); + } + } + } + + const Node *Find(const std::vector<FactorType> &factors, const WORD &word) const + { + typename Children::const_iterator iter = m_children.find(word.hash(factors)); + if (iter == m_children.end()) { + return NULL; + } + else { + const Node &child = iter->second; + return &child; + } + } + + const TPS *GetTargetPhrases() const + { return m_targetPhrases; } + + void SortAndPrune(size_t tableLimit, MemPool &pool, System &system) + { + BOOST_FOREACH(typename Children::value_type &val, m_children){ + Node &child = val.second; + child.SortAndPrune(tableLimit, pool, system); + } + + // prune target phrases in this node + if (m_unsortedTPS) { + m_targetPhrases = new (pool.Allocate<TPS>()) TPS(pool, m_unsortedTPS->size()); + + for (size_t i = 0; i < m_unsortedTPS->size(); ++i) { + TP *tp = (*m_unsortedTPS)[i]; + m_targetPhrases->AddTargetPhrase(*tp); + } + + m_targetPhrases->SortAndPrune(tableLimit); + system.featureFunctions.EvaluateAfterTablePruning(system.GetSystemPool(), *m_targetPhrases, *m_source); + + delete m_unsortedTPS; + } + } + + const Children &GetChildren() const + { return m_children; } + + void Debug(std::ostream &out, const System &system) const { + BOOST_FOREACH(const typename Children::value_type &valPair, m_children) { + const WORD &word = valPair.first; + //std::cerr << word << "(" << word.hash() << ") "; + } + } +protected: + Children m_children; + TPS *m_targetPhrases; + Phrase<WORD> *m_source; + std::vector<TP*> *m_unsortedTPS; + + Node &AddRule(const std::vector<FactorType> &factors, SP &source, TP *target, size_t pos) + { + if (pos == source.GetSize()) { + if (m_unsortedTPS == NULL) { + m_unsortedTPS = new std::vector<TP*>(); + m_source = &source; + } + + m_unsortedTPS->push_back(target); + return *this; + } + else { + const WORD &word = source[pos]; + Node &child = m_children[word.hash(factors)]; + //std::cerr << "added " << word << " " << &child << " from " << this << std::endl; + + return child.AddRule(factors, source, target, pos + 1); + } + } + +}; + + +} +} // namespace + diff --git a/moses2/TranslationModel/Memory/PhraseTableMemory.cpp b/moses2/TranslationModel/Memory/PhraseTableMemory.cpp new file mode 100644 index 000000000..09eead137 --- /dev/null +++ b/moses2/TranslationModel/Memory/PhraseTableMemory.cpp @@ -0,0 +1,268 @@ +/* + * PhraseTableMemory.cpp + * + * Created on: 28 Oct 2015 + * Author: hieu + */ + +#include <cassert> +#include <boost/foreach.hpp> +#include "PhraseTableMemory.h" +#include "../../PhraseBased/PhraseImpl.h" +#include "../../Phrase.h" +#include "../../System.h" +#include "../../Scores.h" +#include "../../InputPathsBase.h" +#include "../../legacy/InputFileStream.h" +#include "util/exception.hh" + +#include "../../PhraseBased/InputPath.h" +#include "../../PhraseBased/TargetPhraseImpl.h" +#include "../../PhraseBased/TargetPhrases.h" + +#include "../../SCFG/PhraseImpl.h" +#include "../../SCFG/TargetPhraseImpl.h" +#include "../../SCFG/InputPath.h" +#include "../../SCFG/Stack.h" +#include "../../SCFG/Stacks.h" +#include "../../SCFG/Manager.h" + + +using namespace std; + +namespace Moses2 +{ + + +//////////////////////////////////////////////////////////////////////// + +PhraseTableMemory::PhraseTableMemory(size_t startInd, const std::string &line) +:PhraseTable(startInd, line) +,m_rootPb(NULL) +,m_rootSCFG(NULL) +{ + ReadParameters(); +} + +PhraseTableMemory::~PhraseTableMemory() +{ + delete m_rootPb; + delete m_rootSCFG; +} + +void PhraseTableMemory::Load(System &system) +{ + FactorCollection &vocab = system.GetVocab(); + MemPool &systemPool = system.GetSystemPool(); + MemPool tmpSourcePool; + + if (system.isPb) { + m_rootPb = new PBNODE(); + } + else { + m_rootSCFG = new SCFGNODE(); + //cerr << "m_rootSCFG=" << m_rootSCFG << endl; + } + + vector<string> toks; + size_t lineNum = 0; + InputFileStream strme(m_path); + string line; + while (getline(strme, line)) { + if (++lineNum % 1000000 == 0) { + cerr << lineNum << " "; + } + toks.clear(); + TokenizeMultiCharSeparator(toks, line, "|||"); + UTIL_THROW_IF2(toks.size() < 3, "Wrong format"); + //cerr << "line=" << line << endl; + //cerr << "system.isPb=" << system.isPb << endl; + + if (system.isPb) { + PhraseImpl *source = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, + toks[0]); + //cerr << "created soure" << endl; + TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(systemPool, *this, system, + toks[1]); + //cerr << "created target" << endl; + target->GetScores().CreateFromString(toks[2], *this, system, true); + //cerr << "created scores:" << *target << endl; + + if (toks.size() >= 4) { + //cerr << "alignstr=" << toks[3] << endl; + target->SetAlignmentInfo(toks[3]); + } + + // properties + if (toks.size() == 7) { + //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); + //strcpy(target->properties, toks[6].c_str()); + } + + system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, + *target); + //cerr << "EvaluateInIsolation:" << *target << endl; + m_rootPb->AddRule(m_input, *source, target); + + //cerr << "target=" << target->Debug(system) << endl; + } + else { + SCFG::PhraseImpl *source = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, + toks[0]); + //cerr << "created source:" << *source << endl; + SCFG::TargetPhraseImpl *target = SCFG::TargetPhraseImpl::CreateFromString(systemPool, *this, + system, toks[1]); + + //cerr << "created target " << *target << " source=" << *source << endl; + + target->GetScores().CreateFromString(toks[2], *this, system, true); + //cerr << "created scores:" << *target << endl; + + //vector<SCORE> scores = Tokenize<SCORE>(toks[2]); + //target->sortScore = (scores.size() >= 3) ? TransformScore(scores[2]) : 0; + + target->SetAlignmentInfo(toks[3]); + + // properties + if (toks.size() == 7) { + //target->properties = (char*) system.systemPool.Allocate(toks[6].size() + 1); + //strcpy(target->properties, toks[6].c_str()); + } + + system.featureFunctions.EvaluateInIsolation(systemPool, system, *source, + *target); + //cerr << "EvaluateInIsolation:" << *target << endl; + m_rootSCFG->AddRule(m_input, *source, target); + } + } + + if (system.isPb) { + m_rootPb->SortAndPrune(m_tableLimit, systemPool, system); + //cerr << "root=" << &m_rootPb << endl; + } + else { + m_rootSCFG->SortAndPrune(m_tableLimit, systemPool, system); + //cerr << "root=" << &m_rootPb << endl; + } + /* + BOOST_FOREACH(const PtMem::Node<Word>::Children::value_type &valPair, m_rootPb.GetChildren()) { + const Word &word = valPair.first; + cerr << word << " "; + } + cerr << endl; + */ +} + +TargetPhrases* PhraseTableMemory::Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const +{ + const SubPhrase<Moses2::Word> &phrase = inputPath.subPhrase; + TargetPhrases *tps = m_rootPb->Find(m_input, phrase); + return tps; +} + +void PhraseTableMemory::InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const +{ + size_t ptInd = GetPtInd(); + ActiveChartEntryMem *chartEntry = new (pool.Allocate<ActiveChartEntryMem>()) ActiveChartEntryMem(pool, *m_rootSCFG); + path.AddActiveChartEntry(ptInd, chartEntry); + //cerr << "InitActiveChart=" << path << endl; +} + +void PhraseTableMemory::Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const +{ + if (path.range.GetNumWordsCovered() > maxChartSpan) { + return; + } + + size_t endPos = path.range.GetEndPos(); + + const SCFG::InputPath *prevPath = static_cast<const SCFG::InputPath*>(path.prefixPath); + UTIL_THROW_IF2(prevPath == NULL, "prefixPath == NULL"); + + // TERMINAL + const SCFG::Word &lastWord = path.subPhrase.Back(); + + const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(endPos, 1); + + //cerr << "BEFORE LookupGivenWord=" << *prevPath << endl; + LookupGivenWord(pool, mgr, *prevPath, lastWord, NULL, subPhrasePath.range, path); + //cerr << "AFTER LookupGivenWord=" << *prevPath << endl; + + // NON-TERMINAL + //const SCFG::InputPath *prefixPath = static_cast<const SCFG::InputPath*>(path.prefixPath); + while (prevPath) { + const Range &prevRange = prevPath->range; + //cerr << "prevRange=" << prevRange << endl; + + size_t startPos = prevRange.GetEndPos() + 1; + size_t ntSize = endPos - startPos + 1; + const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(startPos, ntSize); + + LookupNT(pool, mgr, subPhrasePath.range, *prevPath, stacks, path); + + prevPath = static_cast<const SCFG::InputPath*>(prevPath->prefixPath); + } +} + +void PhraseTableMemory::LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const +{ + const ActiveChartEntryMem &prevEntryCast = static_cast<const ActiveChartEntryMem&>(prevEntry); + + const SCFGNODE &prevNode = prevEntryCast.node; + UTIL_THROW_IF2(&prevNode == NULL, "node == NULL"); + + size_t ptInd = GetPtInd(); + const SCFGNODE *nextNode = prevNode.Find(m_input, wordSought); + + /* + if (outPath.range.GetStartPos() == 1 || outPath.range.GetStartPos() == 2) { + cerr << "range=" << outPath.range + << " prevEntry=" << prevEntry.GetSymbolBind().Debug(mgr.system) + << " wordSought=" << wordSought.Debug(mgr.system) + << " nextNode=" << nextNode + << endl; + } + */ + if (nextNode) { + // new entries + ActiveChartEntryMem *chartEntry = new (pool.Allocate<ActiveChartEntryMem>()) ActiveChartEntryMem(pool, *nextNode, prevEntry); + + chartEntry->AddSymbolBindElement(subPhraseRange, wordSought, hypos, *this); + //cerr << "AFTER Add=" << symbolBind << endl; + + outPath.AddActiveChartEntry(ptInd, chartEntry); + + const SCFG::TargetPhrases *tps = nextNode->GetTargetPhrases(); + if (tps) { + // there are some rules + /* + cerr << "outPath=" << outPath.range + << " bind=" << chartEntry->GetSymbolBind().Debug(mgr.system) + << " pt=" << GetPtInd() + << " tps=" << tps->Debug(mgr.system) << endl; + */ + outPath.AddTargetPhrasesToPath(pool, mgr.system, *this, *tps, chartEntry->GetSymbolBind()); + + } + + //cerr << "AFTER outPath=" << outPath << endl; + } +} + +} + diff --git a/moses2/TranslationModel/Memory/PhraseTableMemory.h b/moses2/TranslationModel/Memory/PhraseTableMemory.h new file mode 100644 index 000000000..035c7c9c5 --- /dev/null +++ b/moses2/TranslationModel/Memory/PhraseTableMemory.h @@ -0,0 +1,85 @@ +/* + * PhraseTableMemory.h + * + * Created on: 28 Oct 2015 + * Author: hieu + */ +#pragma once + +#include "../PhraseTable.h" +#include "../../legacy/Util2.h" +#include "../../SCFG/InputPath.h" +#include "Node.h" +#include "../../PhraseBased/PhraseImpl.h" +#include "../../PhraseBased/TargetPhraseImpl.h" +#include "../../PhraseBased/TargetPhrases.h" +#include "../../SCFG/PhraseImpl.h" +#include "../../SCFG/TargetPhraseImpl.h" +#include "../../SCFG/TargetPhrases.h" + +namespace Moses2 +{ + +class PhraseTableMemory: public PhraseTable +{ + typedef PtMem::Node<Word, Phrase<Word>, TargetPhraseImpl, TargetPhrases> PBNODE; + typedef PtMem::Node<SCFG::Word, Phrase<SCFG::Word>, SCFG::TargetPhraseImpl, SCFG::TargetPhrases> SCFGNODE; + +////////////////////////////////////// + class ActiveChartEntryMem : public SCFG::ActiveChartEntry + { + typedef SCFG::ActiveChartEntry Parent; + public: + const PhraseTableMemory::SCFGNODE &node; + + ActiveChartEntryMem(MemPool &pool, const PhraseTableMemory::SCFGNODE &vnode) + :Parent(pool) + ,node(vnode) + {} + + ActiveChartEntryMem( + MemPool &pool, + const PhraseTableMemory::SCFGNODE &vnode, + const ActiveChartEntry &prevEntry) + :Parent(prevEntry) + ,node(vnode) + {} + }; + + ////////////////////////////////////// +public: + PhraseTableMemory(size_t startInd, const std::string &line); + virtual ~PhraseTableMemory(); + + virtual void Load(System &system); + virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const; + + virtual void InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; + + void Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; + +protected: + PBNODE *m_rootPb; + SCFGNODE *m_rootSCFG; + + void LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; + +}; + +} + diff --git a/moses2/TranslationModel/PhraseTable.cpp b/moses2/TranslationModel/PhraseTable.cpp new file mode 100644 index 000000000..c790147bb --- /dev/null +++ b/moses2/TranslationModel/PhraseTable.cpp @@ -0,0 +1,183 @@ +/* + * PhraseTable.cpp + * + * Created on: 23 Oct 2015 + * Author: hieu + */ +#include <boost/foreach.hpp> +#include <queue> +#include "PhraseTable.h" +#include "../legacy/Util2.h" +#include "../TypeDef.h" +#include "../InputType.h" +#include "../PhraseBased/Manager.h" +#include "../PhraseBased/InputPath.h" +#include "../SCFG/InputPath.h" +#include "../SCFG/Manager.h" + +using namespace std; + +namespace Moses2 +{ + +//////////////////////////////////////////////////////////////////////////// +PhraseTable::PhraseTable(size_t startInd, const std::string &line) : + StatelessFeatureFunction(startInd, line), m_tableLimit(20) // default + , m_maxCacheSize(DEFAULT_MAX_TRANS_OPT_CACHE_SIZE) +{ + m_input.push_back(0); +} + +PhraseTable::~PhraseTable() +{ + // TODO Auto-generated destructor stub +} + +void PhraseTable::SetParameter(const std::string& key, const std::string& value) +{ + if (key == "cache-size") { + m_maxCacheSize = Scan<size_t>(value); + } + else if (key == "path") { + m_path = value; + } + else if (key == "input-factor") { + m_input = Tokenize<FactorType>(value, ","); + } + else if (key == "output-factor") { + m_output = Tokenize<FactorType>(value, ","); + } + else if (key == "table-limit") { + m_tableLimit = Scan<size_t>(value); + } + else { + StatelessFeatureFunction::SetParameter(key, value); + } +} + +bool PhraseTable::SatisfyBackoff(const Manager &mgr, const InputPath &path) const +{ + const InputType &input = mgr.GetInput(); + if ((mgr.system.options.input.xml_policy == XmlExclusive) + && input.XmlOverlap(path.range.GetStartPos(), path.range.GetEndPos())) { + return false; + } + + //cerr << GetName() << "=" << GetPtInd() << "=" << decodeGraphBackoff << endl; + if (decodeGraphBackoff == 0) { + // always lookup + return true; + } + else if (decodeGraphBackoff == -1) { + // lookup only if there's no existing rules + return path.GetNumRules() ? false : true; + } + else if (path.range.GetNumWordsCovered() <= decodeGraphBackoff) { + return path.GetNumRules() ? false : true; + } + + return false; +} + +void PhraseTable::Lookup(const Manager &mgr, InputPathsBase &inputPaths) const +{ + BOOST_FOREACH(InputPathBase *pathBase, inputPaths){ + InputPath *path = static_cast<InputPath*>(pathBase); + //cerr << "path=" << path->range << " "; + + if (SatisfyBackoff(mgr, *path)) { + TargetPhrases *tpsPtr = Lookup(mgr, mgr.GetPool(), *path); + /* + cerr << "tpsPtr=" << tpsPtr << " "; + if (tps.get()) { + cerr << tps.get()->GetSize(); + } + cerr << endl; + */ + + path->AddTargetPhrases(*this, tpsPtr); + } + } + +} + +TargetPhrases *PhraseTable::Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const +{ + UTIL_THROW2("Not implemented"); +} + +void PhraseTable::EvaluateInIsolation(MemPool &pool, const System &system, + const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const +{ +} + +void PhraseTable::EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source, + const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores, + SCORE &estimatedScore) const +{ + +} + +// scfg +void PhraseTable::LookupUnary(MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const +{ + //cerr << "BEFORE LookupUnary" << path.Debug(mgr.system) << endl; + size_t startPos = path.range.GetStartPos(); + const SCFG::InputPath *prevPath = mgr.GetInputPaths().GetMatrix().GetValue(startPos, 0); + LookupNT(pool, mgr, path.range, *prevPath, stacks, path); + //cerr << "AFTER LookupUnary" << path.Debug(mgr.system) << endl; +} + +void PhraseTable::LookupNT( + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const +{ + size_t endPos = outPath.range.GetEndPos(); + + const Range &prevRange = prevPath.range; + + size_t startPos = prevRange.GetEndPos() + 1; + size_t ntSize = endPos - startPos + 1; + + const SCFG::Stack &ntStack = stacks.GetStack(startPos, ntSize); + const SCFG::Stack::Coll &stackColl = ntStack.GetColl(); + + BOOST_FOREACH (const SCFG::Stack::Coll::value_type &valPair, stackColl) { + const SCFG::Word &ntSought = valPair.first; + const Moses2::HypothesisColl *hypos = valPair.second; + const Moses2::Hypotheses &sortedHypos = hypos->GetSortedAndPrunedHypos(mgr, mgr.arcLists); + //cerr << "ntSought=" << ntSought << ntSought.isNonTerminal << endl; + LookupGivenWord(pool, mgr, prevPath, ntSought, &sortedHypos, subPhraseRange, outPath); + } +} + +void PhraseTable::LookupGivenWord( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const +{ + size_t ptInd = GetPtInd(); + + + BOOST_FOREACH(const SCFG::ActiveChartEntry *prevEntry, prevPath.GetActiveChart(ptInd).entries) { + //cerr << "BEFORE LookupGivenNode=" << prevPath << endl; + LookupGivenNode(pool, mgr, *prevEntry, wordSought, hypos, subPhraseRange, outPath); + //cerr << "AFTER LookupGivenNode=" << prevPath << endl; + } +} + +} + diff --git a/moses2/TranslationModel/PhraseTable.h b/moses2/TranslationModel/PhraseTable.h new file mode 100644 index 000000000..9237f5ba6 --- /dev/null +++ b/moses2/TranslationModel/PhraseTable.h @@ -0,0 +1,128 @@ +/* + * PhraseTable.h + * + * Created on: 23 Oct 2015 + * Author: hieu + */ +#pragma once +#include <string> +#include <boost/unordered_map.hpp> +#include "../Word.h" +#include "../HypothesisColl.h" +#include "../FF/StatelessFeatureFunction.h" +#include "../legacy/Util2.h" + +namespace Moses2 +{ + +class System; +class InputPathsBase; +class InputPath; +class Manager; +class TargetPhrases; +class Range; + +namespace SCFG +{ +class InputPath; +class Stacks; +class Manager; +class ActiveChartEntry; +} + +//////////////////////////////////////////////////////////////////////// +class PhraseTable: public StatelessFeatureFunction +{ +public: + int decodeGraphBackoff; + + PhraseTable(size_t startInd, const std::string &line); + virtual ~PhraseTable(); + + virtual void SetParameter(const std::string& key, const std::string& value); + virtual void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const; + virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const; + + void SetPtInd(size_t ind) + { m_ptInd = ind; } + + size_t GetPtInd() const + { return m_ptInd; } + + bool SatisfyBackoff(const Manager &mgr, const InputPath &path) const; + + virtual void + EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<Moses2::Word> &source, + const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const; + + virtual void + EvaluateInIsolation(MemPool &pool, const System &system, const Phrase<SCFG::Word> &source, + const TargetPhrase<SCFG::Word> &targetPhrase, Scores &scores, + SCORE &estimatedScore) const; + + // scfg + virtual void InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const = 0; + + virtual void Lookup( + MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const = 0; + + virtual void LookupUnary(MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; + +protected: + std::string m_path; + size_t m_ptInd; // in the order that it is list in [feature], NOT order of [mapping] + size_t m_tableLimit; + std::vector<FactorType> m_input, m_output; + + // cache + size_t m_maxCacheSize; // 0 = no caching + + struct CacheCollEntry2 + { + TargetPhrases *tpsPtr; + clock_t clock; + }; + + // scfg + virtual void LookupNT( + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const; + + virtual void LookupGivenWord( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; + + virtual void LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const = 0; + +}; + +} + diff --git a/moses2/TranslationModel/ProbingPT/ProbingPT.cpp b/moses2/TranslationModel/ProbingPT/ProbingPT.cpp new file mode 100644 index 000000000..1f22f45be --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/ProbingPT.cpp @@ -0,0 +1,756 @@ +/* + * ProbingPT.cpp + * + * Created on: 3 Nov 2015 + * Author: hieu + */ +#include <boost/foreach.hpp> +#include "ProbingPT.h" +#include "querying.hh" +#include "probing_hash_utils.hh" +#include "util/exception.hh" +#include "../../System.h" +#include "../../Scores.h" +#include "../../Phrase.h" +#include "../../legacy/InputFileStream.h" +#include "../../legacy/FactorCollection.h" +#include "../../legacy/Util2.h" +#include "../../FF/FeatureFunctions.h" +#include "../../PhraseBased/PhraseImpl.h" +#include "../../PhraseBased/TargetPhraseImpl.h" +#include "../../PhraseBased/Manager.h" +#include "../../PhraseBased/TargetPhrases.h" +#include "../../SCFG/InputPath.h" +#include "../../SCFG/Manager.h" +#include "../../SCFG/TargetPhraseImpl.h" +#include "../../SCFG/PhraseImpl.h" + +using namespace std; + +namespace Moses2 +{ +ProbingPT::ActiveChartEntryProbing::ActiveChartEntryProbing( + MemPool &pool, + const ActiveChartEntryProbing &prevEntry) +:Parent(prevEntry) +,m_key(prevEntry.m_key) +{} + +void ProbingPT::ActiveChartEntryProbing::AddSymbolBindElement( + const Range &range, + const SCFG::Word &word, + const Moses2::Hypotheses *hypos, + const Moses2::PhraseTable &pt) +{ + const ProbingPT &probingPt = static_cast<const ProbingPT&>(pt); + std::pair<bool, uint64_t> key = GetKey(word, probingPt); + UTIL_THROW_IF2(!key.first, "Word should have been in source vocab"); + m_key = key.second; + + ActiveChartEntry::AddSymbolBindElement(range, word, hypos, pt); +} + +std::pair<bool, uint64_t> ProbingPT::ActiveChartEntryProbing::GetKey(const SCFG::Word &nextWord, const ProbingPT &pt) const +{ + std::pair<bool, uint64_t> ret; + ret.second = m_key; + uint64_t probingId = pt.GetSourceProbingId(nextWord); + if (probingId == pt.GetUnk()) { + ret.first = false; + return ret; + } + + ret.first = true; + size_t phraseSize = m_symbolBind.coll.size(); + ret.second += probingId << phraseSize; + return ret; +} + +//////////////////////////////////////////////////////////////////////////// +ProbingPT::ProbingPT(size_t startInd, const std::string &line) +:PhraseTable(startInd, line) +,load_method(util::POPULATE_OR_READ) +{ + ReadParameters(); +} + +ProbingPT::~ProbingPT() +{ + delete m_engine; +} + +void ProbingPT::Load(System &system) +{ + m_engine = new QueryEngine(m_path.c_str(), load_method); + + m_unkId = 456456546456; + + FactorCollection &vocab = system.GetVocab(); + + // source vocab + const std::map<uint64_t, std::string> &sourceVocab = + m_engine->getSourceVocab(); + std::map<uint64_t, std::string>::const_iterator iterSource; + for (iterSource = sourceVocab.begin(); iterSource != sourceVocab.end(); + ++iterSource) { + string wordStr = iterSource->second; + bool isNT; + //cerr << "wordStr=" << wordStr << endl; + ReformatWord(system, wordStr, isNT); + //cerr << "wordStr=" << wordStr << endl; + + const Factor *factor = vocab.AddFactor(wordStr, system, isNT); + + uint64_t probingId = iterSource->first; + size_t factorId = factor->GetId(); + + if (factorId >= m_sourceVocab.size()) { + m_sourceVocab.resize(factorId + 1, m_unkId); + } + m_sourceVocab[factorId] = probingId; + } + + // target vocab + InputFileStream targetVocabStrme(m_path + "/TargetVocab.dat"); + string line; + while (getline(targetVocabStrme, line)) { + vector<string> toks = Tokenize(line, "\t"); + UTIL_THROW_IF2(toks.size() != 2, string("Incorrect format:") + line + "\n"); + + bool isNT; + //cerr << "wordStr=" << toks[0] << endl; + ReformatWord(system, toks[0], isNT); + //cerr << "wordStr=" << toks[0] << endl; + + const Factor *factor = vocab.AddFactor(toks[0], system, isNT); + uint32_t probingId = Scan<uint32_t>(toks[1]); + + if (probingId >= m_targetVocab.size()) { + m_targetVocab.resize(probingId + 1); + } + + std::pair<bool, const Factor*> ele(isNT, factor); + m_targetVocab[probingId] = ele; + } + + // alignments + CreateAlignmentMap(system, m_path + "/Alignments.dat"); + + // cache + CreateCache(system); +} + +void ProbingPT::SetParameter(const std::string& key, const std::string& value) +{ + if (key == "load") { + if (value == "lazy") { + load_method = util::LAZY; + } + else if (value == "populate_or_lazy") { + load_method = util::POPULATE_OR_LAZY; + } + else if (value == "populate_or_read" || value == "populate") { + load_method = util::POPULATE_OR_READ; + } + else if (value == "read") { + load_method = util::READ; + } + else if (value == "parallel_read") { + load_method = util::PARALLEL_READ; + } + else { + UTIL_THROW2("load method not supported" << value); + } + } + else { + PhraseTable::SetParameter(key, value); + } +} + +void ProbingPT::CreateAlignmentMap(System &system, const std::string path) +{ + const std::vector< std::vector<unsigned char> > &probingAlignColl = m_engine->getAlignments(); + m_aligns.resize(probingAlignColl.size(), NULL); + + for (size_t i = 0; i < probingAlignColl.size(); ++i) { + AlignmentInfo::CollType aligns; + + const std::vector<unsigned char> &probingAligns = probingAlignColl[i]; + for (size_t j = 0; j < probingAligns.size(); j += 2) { + size_t startPos = probingAligns[j]; + size_t endPos = probingAligns[j+1]; + //cerr << "startPos=" << startPos << " " << endPos << endl; + aligns.insert(std::pair<size_t,size_t>(startPos, endPos)); + } + + const AlignmentInfo *align = AlignmentInfoCollection::Instance().Add(aligns); + m_aligns[i] = align; + //cerr << "align=" << align->Debug(system) << endl; + } +} + +void ProbingPT::Lookup(const Manager &mgr, InputPathsBase &inputPaths) const +{ + BOOST_FOREACH(InputPathBase *pathBase, inputPaths){ + InputPath *path = static_cast<InputPath*>(pathBase); + + if (SatisfyBackoff(mgr, *path)) { + TargetPhrases *tpsPtr; + tpsPtr = Lookup(mgr, mgr.GetPool(), *path); + path->AddTargetPhrases(*this, tpsPtr); + } + } +} + +TargetPhrases* ProbingPT::Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const +{ + /* + if (inputPath.prefixPath && inputPath.prefixPath->GetTargetPhrases(*this) == NULL) { + // assume all paths have prefixes, except rules with 1 word source + return NULL; + } + else { + const Phrase &sourcePhrase = inputPath.subPhrase; + std::pair<TargetPhrases*, uint64_t> tpsAndKey = CreateTargetPhrase(pool, mgr.system, sourcePhrase); + return tpsAndKey.first; + } + */ + const Phrase<Moses2::Word> &sourcePhrase = inputPath.subPhrase; + + // get hash for source phrase + std::pair<bool, uint64_t> keyStruct = GetKey(sourcePhrase); + if (!keyStruct.first) { + return NULL; + } + + // check in cache + CachePb::const_iterator iter = m_cachePb.find(keyStruct.second); + if (iter != m_cachePb.end()) { + //cerr << "FOUND IN CACHE " << keyStruct.second << " " << sourcePhrase.Debug(mgr.system) << endl; + TargetPhrases *tps = iter->second; + return tps; + } + + // query pt + TargetPhrases *tps = CreateTargetPhrases(pool, mgr.system, sourcePhrase, + keyStruct.second); + return tps; +} + +std::pair<bool, uint64_t> ProbingPT::GetKey(const Phrase<Moses2::Word> &sourcePhrase) const +{ + std::pair<bool, uint64_t> ret; + + // create a target phrase from the 1st word of the source, prefix with 'ProbingPT:' + size_t sourceSize = sourcePhrase.GetSize(); + assert(sourceSize); + + uint64_t probingSource[sourceSize]; + GetSourceProbingIds(sourcePhrase, ret.first, probingSource); + if (!ret.first) { + // source phrase contains a word unknown in the pt. + // We know immediately there's no translation for it + } + else { + ret.second = m_engine->getKey(probingSource, sourceSize); + } + + return ret; + +} + +TargetPhrases *ProbingPT::CreateTargetPhrases(MemPool &pool, + const System &system, const Phrase<Moses2::Word> &sourcePhrase, uint64_t key) const +{ + TargetPhrases *tps = NULL; + + //Actual lookup + std::pair<bool, uint64_t> query_result; // 1st=found, 2nd=target file offset + query_result = m_engine->query(key); + //cerr << "key2=" << query_result.second << endl; + + if (query_result.first) { + const char *offset = m_engine->memTPS + query_result.second; + uint64_t *numTP = (uint64_t*) offset; + + tps = new (pool.Allocate<TargetPhrases>()) TargetPhrases(pool, *numTP); + + offset += sizeof(uint64_t); + for (size_t i = 0; i < *numTP; ++i) { + TargetPhraseImpl *tp = CreateTargetPhrase(pool, system, offset); + assert(tp); + const FeatureFunctions &ffs = system.featureFunctions; + ffs.EvaluateInIsolation(pool, system, sourcePhrase, *tp); + + tps->AddTargetPhrase(*tp); + + } + + tps->SortAndPrune(m_tableLimit); + system.featureFunctions.EvaluateAfterTablePruning(pool, *tps, sourcePhrase); + //cerr << *tps << endl; + } + + return tps; +} + +TargetPhraseImpl *ProbingPT::CreateTargetPhrase( + MemPool &pool, + const System &system, + const char *&offset) const +{ + TargetPhraseInfo *tpInfo = (TargetPhraseInfo*) offset; + size_t numRealWords = tpInfo->numWords / m_output.size(); + + TargetPhraseImpl *tp = + new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, *this, + system, numRealWords); + + offset += sizeof(TargetPhraseInfo); + + // scores + SCORE *scores = (SCORE*) offset; + + size_t totalNumScores = m_engine->num_scores + m_engine->num_lex_scores; + + if (m_engine->logProb) { + // set pt score for rule + tp->GetScores().PlusEquals(system, *this, scores); + + // save scores for other FF, eg. lex RO. Just give the offset + if (m_engine->num_lex_scores) { + tp->scoreProperties = scores + m_engine->num_scores; + } + } + else { + // log score 1st + SCORE logScores[totalNumScores]; + for (size_t i = 0; i < totalNumScores; ++i) { + logScores[i] = FloorScore(TransformScore(scores[i])); + } + + // set pt score for rule + tp->GetScores().PlusEquals(system, *this, logScores); + + // save scores for other FF, eg. lex RO. + tp->scoreProperties = pool.Allocate<SCORE>(m_engine->num_lex_scores); + for (size_t i = 0; i < m_engine->num_lex_scores; ++i) { + tp->scoreProperties[i] = logScores[i + m_engine->num_scores]; + } + } + + offset += sizeof(SCORE) * totalNumScores; + + // words + for (size_t targetPos = 0; targetPos < numRealWords; ++targetPos) { + for (size_t i = 0; i < m_output.size(); ++i) { + FactorType factorType = m_output[i]; + + uint32_t *probingId = (uint32_t*) offset; + + const std::pair<bool, const Factor *> *factorPair = GetTargetFactor(*probingId); + assert(factorPair); + assert(!factorPair->first); + + Word &word = (*tp)[targetPos]; + word[factorType] = factorPair->second; + + offset += sizeof(uint32_t); + } + } + + // align + uint32_t alignTerm = tpInfo->alignTerm; + //cerr << "alignTerm=" << alignTerm << endl; + UTIL_THROW_IF2(alignTerm >= m_aligns.size(), "Unknown alignInd"); + tp->Parent::SetAlignTerm(*m_aligns[alignTerm]); + + // properties TODO + + return tp; +} + +void ProbingPT::GetSourceProbingIds(const Phrase<Moses2::Word> &sourcePhrase, + bool &ok, uint64_t probingSource[]) const +{ + + size_t size = sourcePhrase.GetSize(); + for (size_t i = 0; i < size; ++i) { + const Word &word = sourcePhrase[i]; + uint64_t probingId = GetSourceProbingId(word); + if (probingId == m_unkId) { + ok = false; + return; + } + else { + probingSource[i] = probingId; + } + } + + ok = true; +} + +uint64_t ProbingPT::GetSourceProbingId(const Word &word) const +{ + uint64_t ret = 0; + + for (size_t i = 0; i < m_input.size(); ++i) { + FactorType factorType = m_input[i]; + const Factor *factor = word[factorType]; + + size_t factorId = factor->GetId(); + if (factorId >= m_sourceVocab.size()) { + return m_unkId; + } + ret += m_sourceVocab[factorId]; + } + + return ret; +} + +void ProbingPT::CreateCache(System &system) +{ + if (m_maxCacheSize == 0) { + return; + } + + string filePath = m_path + "/cache"; + InputFileStream strme(filePath); + + string line; + getline(strme, line); + //float totalCount = Scan<float>(line); + + MemPool &pool = system.GetSystemPool(); + FactorCollection &vocab = system.GetVocab(); + + MemPool tmpSourcePool; + + size_t lineCount = 0; + while (getline(strme, line) && lineCount < m_maxCacheSize) { + vector<string> toks = Tokenize(line, "\t"); + assert(toks.size() == 3); + uint64_t key = Scan<uint64_t>(toks[1]); + //cerr << "line=" << line << endl; + + if (system.isPb) { + PhraseImpl *sourcePhrase = PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, toks[2]); + + /* + std::pair<bool, uint64_t> retStruct = GetKey(*sourcePhrase); + if (!retStruct.first) { + UTIL_THROW2("Unknown cache entry"); + } + cerr << "key=" << retStruct.second << " " << key << endl; + */ + TargetPhrases *tps = CreateTargetPhrases(pool, system, *sourcePhrase, key); + assert(tps); + + m_cachePb[key] = tps; + } + else { + // SCFG + SCFG::PhraseImpl *sourcePhrase = SCFG::PhraseImpl::CreateFromString(tmpSourcePool, vocab, system, toks[2], false); + //cerr << "sourcePhrase=" << sourcePhrase->Debug(system) << endl; + + std::pair<bool, SCFG::TargetPhrases*> tpsPair = CreateTargetPhrasesSCFG(pool, system, *sourcePhrase, key); + assert(tpsPair.first && tpsPair.second); + + m_cacheSCFG[key] = tpsPair.second; + } + ++lineCount; + } + +} + +/////////////////////////////////////////////////////////////////////////////// +// SCFG +/////////////////////////////////////////////////////////////////////////////// + +void ProbingPT::ReformatWord(System &system, std::string &wordStr, bool &isNT) +{ + isNT = false; + if (system.isPb) { + return; + } + else { + isNT = (wordStr[0] == '[' && wordStr[wordStr.size() - 1] == ']'); + //cerr << "nt=" << nt << endl; + + if (isNT) { + size_t startPos = wordStr.find("]["); + if (startPos == string::npos) { + startPos = 1; + } + else { + startPos += 2; + } + + wordStr = wordStr.substr(startPos, wordStr.size() - startPos - 1); + //cerr << "wordStr=" << wordStr << endl; + } + } +} + +void ProbingPT::InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const +{ + //cerr << "InitActiveChart=" << path.Debug(cerr, mgr.system) << endl; + size_t ptInd = GetPtInd(); + ActiveChartEntryProbing *chartEntry = new (pool.Allocate<ActiveChartEntryProbing>()) ActiveChartEntryProbing(pool); + path.AddActiveChartEntry(ptInd, chartEntry); +} + +void ProbingPT::Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const +{ + //cerr << "Lookup=" << endl; + if (path.range.GetNumWordsCovered() > maxChartSpan) { + return; + } + + size_t endPos = path.range.GetEndPos(); + + const SCFG::InputPath *prevPath = static_cast<const SCFG::InputPath*>(path.prefixPath); + UTIL_THROW_IF2(prevPath == NULL, "prefixPath == NULL"); + + // TERMINAL + const SCFG::Word &lastWord = path.subPhrase.Back(); + + const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(endPos, 1); + + //cerr << "BEFORE LookupGivenWord=" << *prevPath << endl; + LookupGivenWord(pool, mgr, *prevPath, lastWord, NULL, subPhrasePath.range, path); + //cerr << "AFTER LookupGivenWord=" << *prevPath << endl; + + // NON-TERMINAL + //const SCFG::InputPath *prefixPath = static_cast<const SCFG::InputPath*>(path.prefixPath); + while (prevPath) { + const Range &prevRange = prevPath->range; + //cerr << "prevRange=" << prevRange << endl; + + size_t startPos = prevRange.GetEndPos() + 1; + size_t ntSize = endPos - startPos + 1; + const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(startPos, ntSize); + + LookupNT(pool, mgr, subPhrasePath.range, *prevPath, stacks, path); + + prevPath = static_cast<const SCFG::InputPath*>(prevPath->prefixPath); + } +} + +void ProbingPT::LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const +{ + const ActiveChartEntryProbing &prevEntryCast = static_cast<const ActiveChartEntryProbing&>(prevEntry); + + std::pair<bool, uint64_t> key = prevEntryCast.GetKey(wordSought, *this); + + if (!key.first) { + // should only occasionally happen when looking up unary rules + return; + } + + const Phrase<SCFG::Word> &sourcePhrase = outPath.subPhrase; + + // check in cache + CacheSCFG::const_iterator iter = m_cacheSCFG.find(key.second); + if (iter != m_cacheSCFG.end()) { + //cerr << "FOUND IN CACHE " << key.second << " " << sourcePhrase.Debug(mgr.system) << endl; + SCFG::TargetPhrases *tps = iter->second; + + ActiveChartEntryProbing *chartEntry = new (pool.Allocate<ActiveChartEntryProbing>()) ActiveChartEntryProbing(pool, prevEntryCast); + //cerr << "AFTER chartEntry" << endl; + + chartEntry->AddSymbolBindElement(subPhraseRange, wordSought, hypos, *this); + //cerr << "AFTER AddSymbolBindElement" << endl; + + size_t ptInd = GetPtInd(); + outPath.AddActiveChartEntry(ptInd, chartEntry); + + outPath.AddTargetPhrasesToPath(pool, mgr.system, *this, *tps, chartEntry->GetSymbolBind()); + } + else { + // not in cache. Lookup + std::pair<bool, SCFG::TargetPhrases*> tpsPair = CreateTargetPhrasesSCFG(pool, mgr.system, sourcePhrase, key.second); + assert(tpsPair.first && tpsPair.second); + + if (tpsPair.first) { + // new entries + ActiveChartEntryProbing *chartEntry = new (pool.Allocate<ActiveChartEntryProbing>()) ActiveChartEntryProbing(pool, prevEntryCast); + //cerr << "AFTER chartEntry" << endl; + + chartEntry->AddSymbolBindElement(subPhraseRange, wordSought, hypos, *this); + //cerr << "AFTER AddSymbolBindElement" << endl; + + size_t ptInd = GetPtInd(); + outPath.AddActiveChartEntry(ptInd, chartEntry); + //cerr << "AFTER AddActiveChartEntry" << endl; + + if (tpsPair.second) { + // there are some rules + //cerr << "symbolbind=" << chartEntry->GetSymbolBind().Debug(mgr.system) << endl; + outPath.AddTargetPhrasesToPath(pool, mgr.system, *this, *tpsPair.second, chartEntry->GetSymbolBind()); + } + } + } +} + +SCFG::TargetPhraseImpl *ProbingPT::CreateTargetPhraseSCFG( + MemPool &pool, + const System &system, + const char *&offset) const +{ + TargetPhraseInfo *tpInfo = (TargetPhraseInfo*) offset; + SCFG::TargetPhraseImpl *tp = + new (pool.Allocate<SCFG::TargetPhraseImpl>()) SCFG::TargetPhraseImpl(pool, *this, + system, tpInfo->numWords - 1); + + offset += sizeof(TargetPhraseInfo); + + // scores + SCORE *scores = (SCORE*) offset; + + size_t totalNumScores = m_engine->num_scores + m_engine->num_lex_scores; + + if (m_engine->logProb) { + // set pt score for rule + tp->GetScores().PlusEquals(system, *this, scores); + + // save scores for other FF, eg. lex RO. Just give the offset + if (m_engine->num_lex_scores) { + tp->scoreProperties = scores + m_engine->num_scores; + } + } + else { + // log score 1st + SCORE logScores[totalNumScores]; + for (size_t i = 0; i < totalNumScores; ++i) { + logScores[i] = FloorScore(TransformScore(scores[i])); + } + + // set pt score for rule + tp->GetScores().PlusEquals(system, *this, logScores); + + // save scores for other FF, eg. lex RO. + tp->scoreProperties = pool.Allocate<SCORE>(m_engine->num_lex_scores); + for (size_t i = 0; i < m_engine->num_lex_scores; ++i) { + tp->scoreProperties[i] = logScores[i + m_engine->num_scores]; + } + } + + offset += sizeof(SCORE) * totalNumScores; + + // words + for (size_t i = 0; i < tpInfo->numWords - 1; ++i) { + uint32_t *probingId = (uint32_t*) offset; + + const std::pair<bool, const Factor *> *factorPair = GetTargetFactor(*probingId); + assert(factorPair); + + SCFG::Word &word = (*tp)[i]; + word[0] = factorPair->second; + word.isNonTerminal = factorPair->first; + + offset += sizeof(uint32_t); + } + + // lhs + uint32_t *probingId = (uint32_t*) offset; + + const std::pair<bool, const Factor *> *factorPair = GetTargetFactor(*probingId); + assert(factorPair); + assert(factorPair->first); + + tp->lhs[0] = factorPair->second; + tp->lhs.isNonTerminal = factorPair->first; + + offset += sizeof(uint32_t); + + // align + uint32_t alignTerm = tpInfo->alignTerm; + //cerr << "alignTerm=" << alignTerm << endl; + UTIL_THROW_IF2(alignTerm >= m_aligns.size(), "Unknown alignInd"); + tp->Parent::SetAlignTerm(*m_aligns[alignTerm]); + + uint32_t alignNonTerm = tpInfo->alignNonTerm; + //cerr << "alignTerm=" << alignTerm << endl; + UTIL_THROW_IF2(alignNonTerm >= m_aligns.size(), "Unknown alignInd"); + tp->SetAlignNonTerm(*m_aligns[alignNonTerm]); + + // properties TODO + + return tp; +} + +std::pair<bool, SCFG::TargetPhrases*> ProbingPT::CreateTargetPhrasesSCFG(MemPool &pool, const System &system, + const Phrase<SCFG::Word> &sourcePhrase, uint64_t key) const +{ + std::pair<bool, SCFG::TargetPhrases*> ret(false, NULL); + + std::pair<bool, uint64_t> query_result; // 1st=found, 2nd=target file offset + query_result = m_engine->query(key); + //cerr << "query_result=" << query_result.first << endl; + + /* + if (outPath.range.GetStartPos() == 1 || outPath.range.GetStartPos() == 2) { + cerr << "range=" << outPath.range + << " prevEntry=" << prevEntry.GetSymbolBind().Debug(mgr.system) << " " << prevEntryCast.GetKey() + << " wordSought=" << wordSought.Debug(mgr.system) + << " key=" << key.first << " " << key.second + << " query_result=" << query_result.first << " " << (query_result.second == NONE) + << endl; + } + */ + + if (query_result.first) { + ret.first = true; + size_t ptInd = GetPtInd(); + + if (query_result.second != NONE) { + // there are some rules + const FeatureFunctions &ffs = system.featureFunctions; + + const char *offset = m_engine->memTPS + query_result.second; + uint64_t *numTP = (uint64_t*) offset; + //cerr << "numTP=" << *numTP << endl; + + SCFG::TargetPhrases *tps = new (pool.Allocate<SCFG::TargetPhrases>()) SCFG::TargetPhrases(pool, *numTP); + ret.second = tps; + + offset += sizeof(uint64_t); + for (size_t i = 0; i < *numTP; ++i) { + SCFG::TargetPhraseImpl *tp = CreateTargetPhraseSCFG(pool, system, offset); + assert(tp); + //cerr << "tp=" << tp->Debug(mgr.system) << endl; + + ffs.EvaluateInIsolation(pool, system, sourcePhrase, *tp); + + tps->AddTargetPhrase(*tp); + + } + + tps->SortAndPrune(m_tableLimit); + ffs.EvaluateAfterTablePruning(pool, *tps, sourcePhrase); + //cerr << "tps=" << tps->GetSize() << endl; + + } + } + + return ret; +} + +} // namespace + diff --git a/moses2/TranslationModel/ProbingPT/ProbingPT.h b/moses2/TranslationModel/ProbingPT/ProbingPT.h new file mode 100644 index 000000000..c5fbefd6f --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/ProbingPT.h @@ -0,0 +1,159 @@ +/* + * ProbingPT.h + * + * Created on: 3 Nov 2015 + * Author: hieu + */ + +#pragma once + +#include <boost/iostreams/device/mapped_file.hpp> +#include <boost/thread/tss.hpp> +#include <boost/bimap.hpp> +#include <deque> +#include "../PhraseTable.h" +#include "../../Vector.h" +#include "../../Phrase.h" +#include "../../SCFG/ActiveChart.h" +#include "util/mmap.hh" + +namespace Moses2 +{ +class AlignmentInfo; +class QueryEngine; +class target_text; +class MemPool; +class System; +class RecycleData; + +namespace SCFG +{ +class TargetPhraseImpl; +class TargetPhrases; +} + +class ProbingPT: public Moses2::PhraseTable +{ + ////////////////////////////////////// + class ActiveChartEntryProbing : public SCFG::ActiveChartEntry + { + typedef SCFG::ActiveChartEntry Parent; + public: + + ActiveChartEntryProbing(MemPool &pool) + :Parent(pool) + ,m_key(0) + {} + + ActiveChartEntryProbing( + MemPool &pool, + const ActiveChartEntryProbing &prevEntry); + + uint64_t GetKey() const + { return m_key; } + + std::pair<bool, uint64_t> GetKey(const SCFG::Word &nextWord, const ProbingPT &pt) const; + + virtual void AddSymbolBindElement( + const Range &range, + const SCFG::Word &word, + const Moses2::Hypotheses *hypos, + const Moses2::PhraseTable &pt); + + protected: + uint64_t m_key; + }; + ////////////////////////////////////// + +public: + ProbingPT(size_t startInd, const std::string &line); + virtual ~ProbingPT(); + void Load(System &system); + + virtual void SetParameter(const std::string& key, const std::string& value); + void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const; + + uint64_t GetUnk() const + { return m_unkId; } + + // SCFG + void InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; + + virtual void Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; + + +protected: + std::vector<uint64_t> m_sourceVocab; // factor id -> pt id + std::vector< std::pair<bool, const Factor*> > m_targetVocab; // pt id -> factor* + std::vector<const AlignmentInfo*> m_aligns; + util::LoadMethod load_method; + + uint64_t m_unkId; + QueryEngine *m_engine; + + void CreateAlignmentMap(System &system, const std::string path); + + TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const; + TargetPhrases *CreateTargetPhrases(MemPool &pool, const System &system, + const Phrase<Moses2::Word> &sourcePhrase, uint64_t key) const; + TargetPhraseImpl *CreateTargetPhrase(MemPool &pool, const System &system, + const char *&offset) const; + + inline const std::pair<bool, const Factor*> *GetTargetFactor(uint32_t probingId) const + { + if (probingId >= m_targetVocab.size()) { + return NULL; + } + return &m_targetVocab[probingId]; + } + + std::pair<bool, uint64_t> GetKey(const Phrase<Moses2::Word> &sourcePhrase) const; + + void GetSourceProbingIds(const Phrase<Moses2::Word> &sourcePhrase, bool &ok, + uint64_t probingSource[]) const; + + uint64_t GetSourceProbingId(const Word &word) const; + + // caching + typedef boost::unordered_map<uint64_t, TargetPhrases*> CachePb; + CachePb m_cachePb; + + typedef boost::unordered_map<uint64_t, SCFG::TargetPhrases*> CacheSCFG; + CacheSCFG m_cacheSCFG; + + void CreateCache(System &system); + + void ReformatWord(System &system, std::string &wordStr, bool &isNT); + + // SCFG + void LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; + + std::pair<bool, SCFG::TargetPhrases*> CreateTargetPhrasesSCFG(MemPool &pool, const System &system, + const Phrase<SCFG::Word> &sourcePhrase, uint64_t key) const; + // return value: 1st = there are actual rules, not just a empty cell for prefix + + SCFG::TargetPhraseImpl *CreateTargetPhraseSCFG( + MemPool &pool, + const System &system, + const char *&offset) const; + + +}; + +} + diff --git a/moses2/TranslationModel/ProbingPT/StoreTarget.cpp b/moses2/TranslationModel/ProbingPT/StoreTarget.cpp new file mode 100644 index 000000000..326aaea5f --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/StoreTarget.cpp @@ -0,0 +1,266 @@ +/* + * StoreTarget.cpp + * + * Created on: 19 Jan 2016 + * Author: hieu + */ +#include <boost/foreach.hpp> +#include "StoreTarget.h" +#include "line_splitter.hh" +#include "probing_hash_utils.hh" +#include "../../legacy/OutputFileStream.h" +#include "../../legacy/Util2.h" + +using namespace std; + +namespace Moses2 +{ + +StoreTarget::StoreTarget(const std::string &basepath) +:m_basePath(basepath) +,m_vocab(basepath + "/TargetVocab.dat") +{ + std::string path = basepath + "/TargetColl.dat"; + m_fileTargetColl.open(path.c_str(), + std::ios::out | std::ios::binary | std::ios::ate | std::ios::trunc); + if (!m_fileTargetColl.is_open()) { + throw "can't create file "; + } + +} + +StoreTarget::~StoreTarget() +{ + assert(m_coll.empty()); + m_fileTargetColl.close(); + + // vocab + m_vocab.Save(); +} + +uint64_t StoreTarget::Save() +{ + uint64_t ret = m_fileTargetColl.tellp(); + + // save to disk + uint64_t numTP = m_coll.size(); + m_fileTargetColl.write((char*) &numTP, sizeof(uint64_t)); + + for (size_t i = 0; i < m_coll.size(); ++i) { + Save(*m_coll[i]); + } + + // clear coll + RemoveAllInColl(m_coll); + m_coll.clear(); + + // starting position of coll + return ret; +} + +void StoreTarget::Save(const target_text &rule) +{ + // metadata for each tp + TargetPhraseInfo tpInfo; + tpInfo.alignTerm = GetAlignId(rule.word_align_term); + tpInfo.alignNonTerm = GetAlignId(rule.word_align_non_term); + tpInfo.numWords = rule.target_phrase.size(); + tpInfo.propLength = rule.property.size(); + + //cerr << "TPInfo=" << sizeof(TPInfo); + m_fileTargetColl.write((char*) &tpInfo, sizeof(TargetPhraseInfo)); + + // scores + for (size_t i = 0; i < rule.prob.size(); ++i) { + float prob = rule.prob[i]; + m_fileTargetColl.write((char*) &prob, sizeof(prob)); + } + + // tp + for (size_t i = 0; i < rule.target_phrase.size(); ++i) { + uint32_t vocabId = rule.target_phrase[i]; + m_fileTargetColl.write((char*) &vocabId, sizeof(vocabId)); + } + + // prop TODO + +} + +void StoreTarget::SaveAlignment() +{ + std::string path = m_basePath + "/Alignments.dat"; + Moses2::OutputFileStream file(path); + + BOOST_FOREACH(Alignments::value_type &valPair, m_aligns) { + file << valPair.second << "\t"; + + const std::vector<size_t> &aligns = valPair.first; + BOOST_FOREACH(size_t align, aligns) { + file << align << " "; + } + file << endl; + } + +} + +void StoreTarget::Append(const line_text &line, bool log_prob, bool scfg) +{ + target_text *rule = new target_text; + //cerr << "line.target_phrase=" << line.target_phrase << endl; + + // target_phrase + vector<bool> nonTerms; + util::TokenIter<util::SingleCharacter> it; + it = util::TokenIter<util::SingleCharacter>(line.target_phrase, + util::SingleCharacter(' ')); + while (it) { + StringPiece word = *it; + //cerr << "word=" << word << endl; + + bool nonTerm = false; + if (scfg) { + // not really sure how to handle factored SCFG and NT + if (scfg && word[0] == '[' && word[word.size() - 1] == ']') { + //cerr << "NON-TERM=" << tok << " " << nonTerms.size() << endl; + nonTerm = true; + } + nonTerms.push_back(nonTerm); + } + + util::TokenIter<util::SingleCharacter> itFactor; + itFactor = util::TokenIter<util::SingleCharacter>(word, + util::SingleCharacter('|')); + while (itFactor) { + StringPiece factor = *itFactor; + + string factorStr = factor.as_string(); + uint32_t vocabId = m_vocab.GetVocabId(factorStr); + + rule->target_phrase.push_back(vocabId); + + itFactor++; + } + + it++; + } + + // probs + it = util::TokenIter<util::SingleCharacter>(line.prob, + util::SingleCharacter(' ')); + while (it) { + string tok = it->as_string(); + float prob = Scan<float>(tok); + + if (log_prob) { + prob = FloorScore(log(prob)); + if (prob == 0.0f) prob = 0.0000000001; + } + + rule->prob.push_back(prob); + it++; + } + + /* + cerr << "nonTerms="; + for (size_t i = 0; i < nonTerms.size(); ++i) { + cerr << nonTerms[i] << " "; + } + cerr << endl; + */ + + // alignment + it = util::TokenIter<util::SingleCharacter>(line.word_align, + util::SingleCharacter(' ')); + while (it) { + string tokPair = Trim(it->as_string()); + if (tokPair.empty()) { + break; + } + + vector<size_t> alignPair = Tokenize<size_t>(tokPair, "-"); + assert(alignPair.size() == 2); + + bool nonTerm = false; + size_t sourcePos = alignPair[0]; + size_t targetPos = alignPair[1]; + if (scfg) { + nonTerm = nonTerms[targetPos]; + } + + //cerr << targetPos << "=" << nonTerm << endl; + + if (nonTerm) { + rule->word_align_non_term.push_back(sourcePos); + rule->word_align_non_term.push_back(targetPos); + //cerr << (int) rule->word_all1.back() << " "; + } + else { + rule->word_align_term.push_back(sourcePos); + rule->word_align_term.push_back(targetPos); + } + + it++; + } + + // extra scores + string prop = line.property.as_string(); + AppendLexRO(prop, rule->prob, log_prob); + + //cerr << "line.property=" << line.property << endl; + //cerr << "prop=" << prop << endl; + + // properties + /* + for (size_t i = 0; i < prop.size(); ++i) { + rule->property.push_back(prop[i]); + } + */ + m_coll.push_back(rule); +} + +uint32_t StoreTarget::GetAlignId(const std::vector<size_t> &align) +{ + boost::unordered_map<std::vector<size_t>, uint32_t>::iterator iter = + m_aligns.find(align); + if (iter == m_aligns.end()) { + uint32_t ind = m_aligns.size(); + m_aligns[align] = ind; + return ind; + } + else { + return iter->second; + } +} + +void StoreTarget::AppendLexRO(std::string &prop, std::vector<float> &retvector, + bool log_prob) const +{ + size_t startPos = prop.find("{{LexRO "); + + if (startPos != string::npos) { + size_t endPos = prop.find("}}", startPos + 8); + string lexProb = prop.substr(startPos + 8, endPos - startPos - 8); + //cerr << "lexProb=" << lexProb << endl; + + // append lex probs to pt probs + vector<float> scores = Tokenize<float>(lexProb); + + if (log_prob) { + for (size_t i = 0; i < scores.size(); ++i) { + scores[i] = FloorScore(log(scores[i])); + if (scores[i] == 0.0f) scores[i] = 0.0000000001; + } + } + + for (size_t i = 0; i < scores.size(); ++i) { + retvector.push_back(scores[i]); + } + + // exclude LexRO property from property column + prop = prop.substr(0, startPos) + + prop.substr(endPos + 2, prop.size() - endPos - 2); + //cerr << "line.property_to_be_binarized=" << line.property_to_be_binarized << "AAAA" << endl; + } +} + +} /* namespace Moses2 */ diff --git a/moses2/TranslationModel/ProbingPT/StoreTarget.h b/moses2/TranslationModel/ProbingPT/StoreTarget.h new file mode 100644 index 000000000..6fc3b1f66 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/StoreTarget.h @@ -0,0 +1,51 @@ +/* + * StoreTarget.h + * + * Created on: 19 Jan 2016 + * Author: hieu + */ +#pragma once +#include <string> +#include <fstream> +#include <vector> +#include <inttypes.h> +#include <boost/unordered_map.hpp> +#include <boost/unordered_set.hpp> +#include "StoreVocab.h" + +namespace Moses2 +{ + +class line_text; +class target_text; + +class StoreTarget +{ +public: + StoreTarget(const std::string &basepath); + virtual ~StoreTarget(); + + uint64_t Save(); + void SaveAlignment(); + + void Append(const line_text &line, bool log_prob, bool scfg); +protected: + std::string m_basePath; + std::fstream m_fileTargetColl; + StoreVocab<uint32_t> m_vocab; + + typedef boost::unordered_map<std::vector<size_t>, uint32_t> Alignments; + Alignments m_aligns; + + std::vector<target_text*> m_coll; + + uint32_t GetAlignId(const std::vector<size_t> &align); + void Save(const target_text &rule); + + void AppendLexRO(std::string &prop, std::vector<float> &retvector, + bool log_prob) const; + +}; + +} /* namespace Moses2 */ + diff --git a/moses2/TranslationModel/ProbingPT/StoreVocab.cpp b/moses2/TranslationModel/ProbingPT/StoreVocab.cpp new file mode 100644 index 000000000..e0b5b0b08 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/StoreVocab.cpp @@ -0,0 +1,13 @@ +/* + * StoreVocab.cpp + * + * Created on: 15 Jun 2016 + * Author: hieu + */ +#include <fstream> +#include "StoreVocab.h" + +namespace Moses2 +{ + +} /* namespace Moses2 */ diff --git a/moses2/TranslationModel/ProbingPT/StoreVocab.h b/moses2/TranslationModel/ProbingPT/StoreVocab.h new file mode 100644 index 000000000..e9808707a --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/StoreVocab.h @@ -0,0 +1,64 @@ +/* + * StoreVocab.h + * + * Created on: 15 Jun 2016 + * Author: hieu + */ +#pragma once +#include <string> +#include <boost/unordered_map.hpp> +#include "../../legacy/OutputFileStream.h" +#include "../../legacy/Util2.h" + +namespace Moses2 +{ + +template<typename VOCABID> +class StoreVocab +{ +protected: + std::string m_path; + + typedef boost::unordered_map<std::string, VOCABID> Coll; + Coll m_vocab; + +public: + StoreVocab(const std::string &path) + :m_path(path) + {} + + virtual ~StoreVocab() {} + + VOCABID GetVocabId(const std::string &word) + { + typename Coll::iterator iter = m_vocab.find(word); + if (iter == m_vocab.end()) { + VOCABID ind = m_vocab.size() + 1; + m_vocab[word] = ind; + return ind; + } + else { + return iter->second; + } + } + + void Insert(VOCABID id, const std::string &word) + { + m_vocab[word] = id; + } + + void Save() + { + OutputFileStream strme(m_path); + + typename Coll::const_iterator iter; + for (iter = m_vocab.begin(); iter != m_vocab.end(); ++iter) { + strme << iter->first << "\t" << iter->second << std::endl; + } + + strme.Close(); + } +}; + +} /* namespace Moses2 */ + diff --git a/moses2/TranslationModel/ProbingPT/hash.cpp b/moses2/TranslationModel/ProbingPT/hash.cpp new file mode 100644 index 000000000..aab5ee2b3 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/hash.cpp @@ -0,0 +1,44 @@ +#include <iostream> +#include "hash.hh" + +using namespace std; + +namespace Moses2 +{ + +uint64_t getHash(StringPiece text) +{ + std::size_t len = text.size(); + uint64_t key = util::MurmurHashNative(text.data(), len); + return key; +} + +std::vector<uint64_t> getVocabIDs(const StringPiece &textin) +{ + //Tokenize + std::vector<uint64_t> output; + + util::TokenIter<util::SingleCharacter> itWord(textin, util::SingleCharacter(' ')); + + while (itWord) { + StringPiece word = *itWord; + uint64_t id = 0; + + util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|')); + while (itFactor) { + StringPiece factor = *itFactor; + //cerr << "factor=" << factor << endl; + + id += getHash(factor); + itFactor++; + } + + output.push_back(id); + itWord++; + } + + return output; +} + +} + diff --git a/moses2/TranslationModel/ProbingPT/hash.hh b/moses2/TranslationModel/ProbingPT/hash.hh new file mode 100644 index 000000000..78cc27999 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/hash.hh @@ -0,0 +1,17 @@ +#pragma once + +#include "util/string_piece.hh" +#include "util/murmur_hash.hh" +#include "util/string_piece.hh" //Tokenization and work with StringPiece +#include "util/tokenize_piece.hh" +#include <vector> + +namespace Moses2 +{ + +//Gets the MurmurmurHash for give string +uint64_t getHash(StringPiece text); + +std::vector<uint64_t> getVocabIDs(const StringPiece &textin); + +} diff --git a/moses2/TranslationModel/ProbingPT/line_splitter.cpp b/moses2/TranslationModel/ProbingPT/line_splitter.cpp new file mode 100644 index 000000000..e4b5e2694 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/line_splitter.cpp @@ -0,0 +1,103 @@ +#include "line_splitter.hh" + +namespace Moses2 +{ + +line_text splitLine(const StringPiece &textin, bool scfg) +{ + const char delim[] = "|||"; + line_text output; + + //Tokenize + util::TokenIter<util::MultiCharacter> it(textin, util::MultiCharacter(delim)); + //Get source phrase + output.source_phrase = Trim(*it); + //std::cerr << "output.source_phrase=" << output.source_phrase << "AAAA" << std::endl; + + //Get target_phrase + it++; + output.target_phrase = Trim(*it); + //std::cerr << "output.target_phrase=" << output.target_phrase << "AAAA" << std::endl; + + if (scfg) { + /* + std::cerr << "output.source_phrase=" << output.source_phrase << std::endl; + std::cerr << "output.target_phrase=" << output.target_phrase << std::endl; + reformatSCFG(output); + std::cerr << "output.source_phrase=" << output.source_phrase << std::endl; + std::cerr << "output.target_phrase=" << output.target_phrase << std::endl; + */ + } + + //Get probabilities + it++; + output.prob = Trim(*it); + //std::cerr << "output.prob=" << output.prob << "AAAA" << std::endl; + + //Get WordAllignment + it++; + if (it == util::TokenIter<util::MultiCharacter>::end()) return output; + output.word_align = Trim(*it); + //std::cerr << "output.word_align=" << output.word_align << "AAAA" << std::endl; + + //Get count + it++; + if (it == util::TokenIter<util::MultiCharacter>::end()) return output; + output.counts = Trim(*it); + //std::cerr << "output.counts=" << output.counts << "AAAA" << std::endl; + + //Get sparse_score + it++; + if (it == util::TokenIter<util::MultiCharacter>::end()) return output; + output.sparse_score = Trim(*it); + //std::cerr << "output.sparse_score=" << output.sparse_score << "AAAA" << std::endl; + + //Get property + it++; + if (it == util::TokenIter<util::MultiCharacter>::end()) return output; + output.property = Trim(*it); + //std::cerr << "output.property=" << output.property << "AAAA" << std::endl; + + return output; +} + +std::vector<unsigned char> splitWordAll1(const StringPiece &textin) +{ + const char delim[] = " "; + const char delim2[] = "-"; + std::vector<unsigned char> output; + + //Case with no word alignments. + if (textin.size() == 0) { + return output; + } + + //Split on space + util::TokenIter<util::MultiCharacter> it(textin, util::MultiCharacter(delim)); + + //For each int + while (it) { + //Split on dash (-) + util::TokenIter<util::MultiCharacter> itInner(*it, + util::MultiCharacter(delim2)); + + //Insert the two entries in the vector. User will read entry 0 and 1 to get the first, + //2 and 3 for second etc. Use unsigned char instead of int to save space, as + //word allignments are all very small numbers that fit in a single byte + output.push_back((unsigned char) (atoi(itInner->data()))); + itInner++; + output.push_back((unsigned char) (atoi(itInner->data()))); + it++; + } + + return output; + +} + +void reformatSCFG(line_text &output) +{ + +} + +} + diff --git a/moses2/TranslationModel/ProbingPT/line_splitter.hh b/moses2/TranslationModel/ProbingPT/line_splitter.hh new file mode 100644 index 000000000..3b086b44a --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/line_splitter.hh @@ -0,0 +1,59 @@ +#pragma once + +#include "util/string_piece.hh" +#include "util/tokenize_piece.hh" +#include "util/file_piece.hh" +#include <vector> +#include <cstdlib> //atof +#include "util/string_piece.hh" //Tokenization and work with StringPiece +#include "util/tokenize_piece.hh" +#include <vector> + +namespace Moses2 +{ + +//Struct for holding processed line +struct line_text +{ + StringPiece source_phrase; + StringPiece target_phrase; + StringPiece prob; + StringPiece word_align; + StringPiece counts; + StringPiece sparse_score; + StringPiece property; + std::string property_to_be_binarized; +}; + +//Struct for holding processed line +struct target_text +{ + std::vector<unsigned int> target_phrase; + std::vector<float> prob; + std::vector<size_t> word_align_term; + std::vector<size_t> word_align_non_term; + std::vector<char> counts; + std::vector<char> sparse_score; + std::vector<char> property; + + /* + void Reset() + { + target_phrase.clear(); + prob.clear(); + word_all1.clear(); + counts.clear(); + sparse_score.clear(); + property.clear(); + } + */ +}; + +//Ask if it's better to have it receive a pointer to a line_text struct +line_text splitLine(const StringPiece &textin, bool scfg); +void reformatSCFG(line_text &output); + +std::vector<unsigned char> splitWordAll1(const StringPiece &textin); + +} + diff --git a/moses2/TranslationModel/ProbingPT/probing_hash_utils.cpp b/moses2/TranslationModel/ProbingPT/probing_hash_utils.cpp new file mode 100644 index 000000000..96c317b65 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/probing_hash_utils.cpp @@ -0,0 +1,40 @@ +#include <iostream> +#include "probing_hash_utils.hh" +#include "util/file.hh" + +namespace Moses2 +{ + +//Read table from disk, return memory map location +char * readTable(const char * filename, util::LoadMethod load_method, util::scoped_fd &file, util::scoped_memory &memory) +{ + //std::cerr << "filename=" << filename << std::endl; + file.reset(util::OpenReadOrThrow(filename)); + uint64_t total_size_ = util::SizeFile(file.get()); + + MapRead(load_method, file.get(), 0, total_size_, memory); + + return (char*) memory.get(); +} + +void serialize_table(char *mem, size_t size, const std::string &filename) +{ + std::ofstream os(filename.c_str(), std::ios::binary); + os.write((const char*) &mem[0], size); + os.close(); + +} + +uint64_t getKey(const uint64_t source_phrase[], size_t size) +{ + //TOO SLOW + //uint64_t key = util::MurmurHashNative(&source_phrase[0], source_phrase.size()); + uint64_t key = 0; + for (size_t i = 0; i < size; i++) { + key += (source_phrase[i] << i); + } + return key; +} + +} + diff --git a/moses2/TranslationModel/ProbingPT/probing_hash_utils.hh b/moses2/TranslationModel/ProbingPT/probing_hash_utils.hh new file mode 100644 index 000000000..368147807 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/probing_hash_utils.hh @@ -0,0 +1,55 @@ +#pragma once + +#include "util/probing_hash_table.hh" + +#include <sys/mman.h> +#include <boost/functional/hash.hpp> +#include <fcntl.h> +#include <fstream> + +namespace Moses2 +{ + +#define API_VERSION 15 + +//Hash table entry +struct Entry +{ + typedef uint64_t Key; + Key key; + + Key GetKey() const + { + return key; + } + + void SetKey(Key to) + { + key = to; + } + + uint64_t value; +}; + +#define NONE std::numeric_limits<uint64_t>::max() + +//Define table +typedef util::ProbingHashTable<Entry, boost::hash<uint64_t> > Table; + +void serialize_table(char *mem, size_t size, const std::string &filename); + +char * readTable(const char * filename, util::LoadMethod load_method, util::scoped_fd &file, util::scoped_memory &memory); + +uint64_t getKey(const uint64_t source_phrase[], size_t size); + +struct TargetPhraseInfo +{ + uint32_t alignTerm; + uint32_t alignNonTerm; + uint16_t numWords; + uint16_t propLength; + uint16_t filler; +}; + +} + diff --git a/moses2/TranslationModel/ProbingPT/querying.cpp b/moses2/TranslationModel/ProbingPT/querying.cpp new file mode 100644 index 000000000..9ea2d8cb6 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/querying.cpp @@ -0,0 +1,180 @@ +#include "querying.hh" +#include "util/exception.hh" +#include "../../legacy/Util2.h" + +using namespace std; + +namespace Moses2 +{ + +QueryEngine::QueryEngine(const char * filepath, util::LoadMethod load_method) +{ + + //Create filepaths + std::string basepath(filepath); + std::string path_to_config = basepath + "/config"; + std::string path_to_hashtable = basepath + "/probing_hash.dat"; + std::string path_to_source_vocabid = basepath + "/source_vocabids"; + std::string alignPath = basepath + "/Alignments.dat"; + + file_exits(basepath); + + ///Source phrase vocabids + read_map(source_vocabids, path_to_source_vocabid.c_str()); + + // alignments + read_alignments(alignPath); + + // target phrase + string targetCollPath = basepath + "/TargetColl.dat"; + memTPS = readTable(targetCollPath.c_str(), load_method, fileTPS_, memoryTPS_); + + //Read config file + boost::unordered_map<std::string, std::string> keyValue; + + std::ifstream config(path_to_config.c_str()); + std::string line; + while (getline(config, line)) { + std::vector<std::string> toks = Moses2::Tokenize(line, "\t"); + UTIL_THROW_IF2(toks.size() != 2, "Wrong config format:" << line); + keyValue[ toks[0] ] = toks[1]; + } + + bool found; + //Check API version: + int version; + found = Get(keyValue, "API_VERSION", version); + if (!found) { + std::cerr << "Old or corrupted version of ProbingPT. Please rebinarize your phrase tables." << std::endl; + } + else if (version != API_VERSION) { + std::cerr << "The ProbingPT API has changed. " << version << "!=" + << API_VERSION << " Please rebinarize your phrase tables." << std::endl; + exit(EXIT_FAILURE); + } + + //Get tablesize. + int tablesize; + found = Get(keyValue, "uniq_entries", tablesize); + if (!found) { + std::cerr << "uniq_entries not found" << std::endl; + exit(EXIT_FAILURE); + } + + //Number of scores + found = Get(keyValue, "num_scores", num_scores); + if (!found) { + std::cerr << "num_scores not found" << std::endl; + exit(EXIT_FAILURE); + } + + //How may scores from lex reordering models + found = Get(keyValue, "num_lex_scores", num_lex_scores); + if (!found) { + std::cerr << "num_lex_scores not found" << std::endl; + exit(EXIT_FAILURE); + } + + // have the scores been log() and FloorScore()? + found = Get(keyValue, "log_prob", logProb); + if (!found) { + std::cerr << "logProb not found" << std::endl; + exit(EXIT_FAILURE); + } + + config.close(); + + //Read hashtable + table_filesize = Table::Size(tablesize, 1.2); + mem = readTable(path_to_hashtable.c_str(), load_method, file_, memory_); + Table table_init(mem, table_filesize); + table = table_init; + + std::cerr << "Initialized successfully! " << std::endl; +} + +QueryEngine::~QueryEngine() +{ + //Clear mmap content from memory. + //munmap(mem, table_filesize); + +} + +uint64_t QueryEngine::getKey(uint64_t source_phrase[], size_t size) const +{ + //TOO SLOW + //uint64_t key = util::MurmurHashNative(&source_phrase[0], source_phrase.size()); + return Moses2::getKey(source_phrase, size); +} + +std::pair<bool, uint64_t> QueryEngine::query(uint64_t key) +{ + std::pair<bool, uint64_t> ret; + + const Entry * entry; + ret.first = table.Find(key, entry); + if (ret.first) { + ret.second = entry->value; + } + return ret; +} + +void QueryEngine::read_alignments(const std::string &alignPath) +{ + std::ifstream strm(alignPath.c_str()); + + string line; + while (getline(strm, line)) { + vector<string> toks = Moses2::Tokenize(line, "\t "); + UTIL_THROW_IF2(toks.size() == 0, "Corrupt alignment file"); + + uint32_t alignInd = Scan<uint32_t>(toks[0]); + if (alignInd >= alignColl.size()) { + alignColl.resize(alignInd + 1); + } + + Alignments &aligns = alignColl[alignInd]; + for (size_t i = 1; i < toks.size(); ++i) { + size_t pos = Scan<size_t>(toks[i]); + aligns.push_back(pos); + } + } +} + +void QueryEngine::file_exits(const std::string &basePath) +{ + if (!FileExists(basePath + "/Alignments.dat")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/Alignments.dat"); + } + if (!FileExists(basePath + "/TargetColl.dat")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/TargetColl.dat"); + } + if (!FileExists(basePath + "/TargetVocab.dat")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/TargetVocab.dat"); + } + if (!FileExists(basePath + "/cache")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/cache"); + } + if (!FileExists(basePath + "/config")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/config"); + } + if (!FileExists(basePath + "/probing_hash.dat")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/probing_hash.dat"); + } + if (!FileExists(basePath + "/source_vocabids")) { + UTIL_THROW2("Require file does not exist in: " << basePath << "/source_vocabids"); + } + + /* + + if (!FileExists(path_to_config) || !FileExists(path_to_hashtable) || + !FileExists(path_to_source_vocabid) || !FileExists(basepath + alignPath) || + !FileExists(basepath + "/TargetColl.dat") || !FileExists(basepath + "/TargetVocab.dat") || + !FileExists(basepath + "/cache")) { + UTIL_THROW2("A required table doesn't exist in: " << basepath); + } + */ +} + +} + diff --git a/moses2/TranslationModel/ProbingPT/querying.hh b/moses2/TranslationModel/ProbingPT/querying.hh new file mode 100644 index 000000000..dcdd2a75a --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/querying.hh @@ -0,0 +1,77 @@ +#pragma once + +#include <boost/iostreams/device/mapped_file.hpp> +#include <boost/unordered_map.hpp> +#include <sys/stat.h> //For finding size of file +#include "vocabid.hh" +#include <algorithm> //toLower +#include <deque> +#include "probing_hash_utils.hh" +#include "hash.hh" //Includes line splitter +#include "line_splitter.hh" +#include "../../legacy/Util2.h" + +namespace Moses2 +{ + +class QueryEngine +{ + std::map<uint64_t, std::string> source_vocabids; + + typedef std::vector<unsigned char> Alignments; + std::vector<Alignments> alignColl; + + Table table; + char *mem; //Memory for the table, necessary so that we can correctly destroy the object + + size_t table_filesize; + bool is_reordering; + + util::scoped_fd file_; + util::scoped_memory memory_; + + // target phrases + boost::iostreams::mapped_file_source file; + + util::scoped_fd fileTPS_; + util::scoped_memory memoryTPS_; + + void read_alignments(const std::string &alignPath); + void file_exits(const std::string &basePath); + +public: + int num_scores; + int num_lex_scores; + bool logProb; + const char *memTPS; + + QueryEngine(const char *, util::LoadMethod load_method); + ~QueryEngine(); + + std::pair<bool, uint64_t> query(uint64_t key); + + const std::map<uint64_t, std::string> &getSourceVocab() const + { return source_vocabids; } + + const std::vector<Alignments> &getAlignments() const + { return alignColl; } + + uint64_t getKey(uint64_t source_phrase[], size_t size) const; + + template<typename T> + inline bool Get(const boost::unordered_map<std::string, std::string> &keyValue, const std::string &sought, T &found) const + { + boost::unordered_map<std::string, std::string>::const_iterator iter = keyValue.find(sought); + if (iter == keyValue.end()) { + return false; + } + + const std::string &foundStr = iter->second; + found = Scan<T>(foundStr); + return true; + } + +}; + +} + diff --git a/moses2/TranslationModel/ProbingPT/storing.cpp b/moses2/TranslationModel/ProbingPT/storing.cpp new file mode 100644 index 000000000..75cdcc038 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/storing.cpp @@ -0,0 +1,303 @@ +#include <sys/stat.h> +#include <boost/foreach.hpp> +#include "line_splitter.hh" +#include "storing.hh" +#include "StoreTarget.h" +#include "StoreVocab.h" +#include "../../legacy/Util2.h" +#include "../../legacy/InputFileStream.h" + +using namespace std; + +namespace Moses2 +{ + +/////////////////////////////////////////////////////////////////////// +void Node::Add(Table &table, const SourcePhrase &sourcePhrase, size_t pos) +{ + if (pos < sourcePhrase.size()) { + uint64_t vocabId = sourcePhrase[pos]; + + Node *child; + Children::iterator iter = m_children.find(vocabId); + if (iter == m_children.end()) { + // New node. Write other children then discard them + BOOST_FOREACH(Children::value_type &valPair, m_children) { + Node &otherChild = valPair.second; + otherChild.Write(table); + } + m_children.clear(); + + // create new node + child = &m_children[vocabId]; + assert(!child->done); + child->key = key + (vocabId << pos); + } + else { + child = &iter->second; + } + + child->Add(table, sourcePhrase, pos + 1); + } + else { + // this node was written previously 'cos it has rules + done = true; + } +} + +void Node::Write(Table &table) +{ + //cerr << "START write " << done << " " << key << endl; + BOOST_FOREACH(Children::value_type &valPair, m_children) { + Node &child = valPair.second; + child.Write(table); + } + + if (!done) { + // save + Entry sourceEntry; + sourceEntry.value = NONE; + sourceEntry.key = key; + + //Put into table + table.Insert(sourceEntry); + } +} + +/////////////////////////////////////////////////////////////////////// +void createProbingPT(const std::string &phrasetable_path, + const std::string &basepath, int num_scores, int num_lex_scores, + bool log_prob, int max_cache_size, bool scfg) +{ + std::cerr << "Starting..." << std::endl; + + //Get basepath and create directory if missing + mkdir(basepath.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH); + + StoreTarget storeTarget(basepath); + + //Get uniq lines: + unsigned long uniq_entries = countUniqueSource(phrasetable_path); + + //Source phrase vocabids + StoreVocab<uint64_t> sourceVocab(basepath + "/source_vocabids"); + + //Read the file + util::FilePiece filein(phrasetable_path.c_str()); + + //Init the probing hash table + size_t size = Table::Size(uniq_entries, 1.2); + char * mem = new char[size]; + memset(mem, 0, size); + Table sourceEntries(mem, size); + + std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> cache; + float totalSourceCount = 0; + + //Keep track of the size of each group of target phrases + size_t line_num = 0; + + //Read everything and processs + std::string prevSource; + + Node sourcePhrases; + sourcePhrases.done = true; + sourcePhrases.key = 0; + + while (true) { + try { + //Process line read + line_text line; + line = splitLine(filein.ReadLine(), scfg); + //cerr << "line=" << line.source_phrase << endl; + + ++line_num; + if (line_num % 1000000 == 0) { + std::cerr << line_num << " " << std::flush; + } + + //Add source phrases to vocabularyIDs + add_to_map(sourceVocab, line.source_phrase); + + if (prevSource.empty()) { + // 1st line + prevSource = line.source_phrase.as_string(); + storeTarget.Append(line, log_prob, scfg); + } + else if (prevSource == line.source_phrase) { + //If we still have the same line, just append to it: + storeTarget.Append(line, log_prob, scfg); + } + else { + assert(prevSource != line.source_phrase); + + //Create a new entry even + + // save + uint64_t targetInd = storeTarget.Save(); + + // next line + storeTarget.Append(line, log_prob, scfg); + + //Create an entry for the previous source phrase: + Entry sourceEntry; + sourceEntry.value = targetInd; + //The key is the sum of hashes of individual words bitshifted by their position in the phrase. + //Probably not entirerly correct, but fast and seems to work fine in practise. + std::vector<uint64_t> vocabid_source = getVocabIDs(prevSource); + if (scfg) { + // storing prefixes? + sourcePhrases.Add(sourceEntries, vocabid_source); + } + sourceEntry.key = getKey(vocabid_source); + + /* + cerr << "prevSource=" << prevSource << flush + << " vocabids=" << Debug(vocabid_source) << flush + << " key=" << sourceEntry.key << endl; + */ + //Put into table + sourceEntries.Insert(sourceEntry); + + // update cache - CURRENT source phrase, not prev + if (max_cache_size) { + std::string countStr = line.counts.as_string(); + countStr = Trim(countStr); + if (!countStr.empty()) { + std::vector<float> toks = Tokenize<float>(countStr); + //cerr << "CACHE:" << line.source_phrase << " " << countStr << " " << toks[1] << endl; + + if (toks.size() >= 2) { + totalSourceCount += toks[1]; + + // compute key for CURRENT source + std::vector<uint64_t> currVocabidSource = getVocabIDs(line.source_phrase.as_string()); + uint64_t currKey = getKey(currVocabidSource); + + CacheItem *item = new CacheItem( + Trim(line.source_phrase.as_string()), + currKey, + toks[1]); + cache.push(item); + + if (max_cache_size > 0 && cache.size() > max_cache_size) { + cache.pop(); + } + } + } + } + + //Set prevLine + prevSource = line.source_phrase.as_string(); + } + + } + catch (util::EndOfFileException e) { + std::cerr + << "Reading phrase table finished, writing remaining files to disk." + << std::endl; + + //After the final entry is constructed we need to add it to the phrase_table + //Create an entry for the previous source phrase: + uint64_t targetInd = storeTarget.Save(); + + Entry sourceEntry; + sourceEntry.value = targetInd; + + //The key is the sum of hashes of individual words. Probably not entirerly correct, but fast + std::vector<uint64_t> vocabid_source = getVocabIDs(prevSource); + sourceEntry.key = getKey(vocabid_source); + + //Put into table + sourceEntries.Insert(sourceEntry); + + break; + } + } + + sourcePhrases.Write(sourceEntries); + + storeTarget.SaveAlignment(); + + serialize_table(mem, size, (basepath + "/probing_hash.dat")); + + sourceVocab.Save(); + + serialize_cache(cache, (basepath + "/cache"), totalSourceCount); + + delete[] mem; + + //Write configfile + std::ofstream configfile; + configfile.open((basepath + "/config").c_str()); + configfile << "API_VERSION\t" << API_VERSION << '\n'; + configfile << "uniq_entries\t" << uniq_entries << '\n'; + configfile << "num_scores\t" << num_scores << '\n'; + configfile << "num_lex_scores\t" << num_lex_scores << '\n'; + configfile << "log_prob\t" << log_prob << '\n'; + configfile.close(); +} + +size_t countUniqueSource(const std::string &path) +{ + size_t ret = 0; + InputFileStream strme(path); + + std::string line, prevSource; + while (std::getline(strme, line)) { + std::vector<std::string> toks = TokenizeMultiCharSeparator(line, "|||"); + assert(toks.size() != 0); + + if (prevSource != toks[0]) { + prevSource = toks[0]; + ++ret; + } + } + + return ret; +} + +void serialize_cache( + std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> &cache, + const std::string &path, float totalSourceCount) +{ + std::vector<const CacheItem*> vec(cache.size()); + + size_t ind = cache.size() - 1; + while (!cache.empty()) { + const CacheItem *item = cache.top(); + vec[ind] = item; + cache.pop(); + --ind; + } + + std::ofstream os(path.c_str()); + + os << totalSourceCount << std::endl; + for (size_t i = 0; i < vec.size(); ++i) { + const CacheItem *item = vec[i]; + os << item->count << "\t" << item->sourceKey << "\t" << item->source << std::endl; + delete item; + } + + os.close(); +} + +uint64_t getKey(const std::vector<uint64_t> &vocabid_source) +{ + return Moses2::getKey(vocabid_source.data(), vocabid_source.size()); +} + +std::vector<uint64_t> CreatePrefix(const std::vector<uint64_t> &vocabid_source, size_t endPos) +{ + assert(endPos < vocabid_source.size()); + + std::vector<uint64_t> ret(endPos + 1); + for (size_t i = 0; i <= endPos; ++i) { + ret[i] = vocabid_source[i]; + } + return ret; +} + +} + diff --git a/moses2/TranslationModel/ProbingPT/storing.hh b/moses2/TranslationModel/ProbingPT/storing.hh new file mode 100644 index 000000000..10d7050d3 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/storing.hh @@ -0,0 +1,95 @@ +#pragma once + +#include <boost/unordered_set.hpp> +#include <boost/unordered_map.hpp> +#include <cstdio> +#include <sstream> +#include <fstream> +#include <iostream> +#include <string> +#include <queue> +#include <sys/stat.h> //mkdir + +#include "hash.hh" //Includes line_splitter +#include "probing_hash_utils.hh" + +#include "util/file_piece.hh" +#include "util/file.hh" +#include "vocabid.hh" + +namespace Moses2 +{ +typedef std::vector<uint64_t> SourcePhrase; + + +class Node +{ + typedef boost::unordered_map<uint64_t, Node> Children; + Children m_children; + +public: + uint64_t key; + bool done; + + Node() + :done(false) + {} + + void Add(Table &table, const SourcePhrase &sourcePhrase, size_t pos = 0); + void Write(Table &table); +}; + + +void createProbingPT(const std::string &phrasetable_path, + const std::string &basepath, int num_scores, int num_lex_scores, + bool log_prob, int max_cache_size, bool scfg); +uint64_t getKey(const std::vector<uint64_t> &source_phrase); + +std::vector<uint64_t> CreatePrefix(const std::vector<uint64_t> &vocabid_source, size_t endPos); + +template<typename T> +std::string Debug(const std::vector<T> &vec) +{ + std::stringstream strm; + for (size_t i = 0; i < vec.size(); ++i) { + strm << vec[i] << " "; + } + return strm.str(); +} + +size_t countUniqueSource(const std::string &path); + +class CacheItem +{ +public: + std::string source; + uint64_t sourceKey; + float count; + CacheItem(const std::string &vSource, uint64_t vSourceKey, float vCount) + :source(vSource) + ,sourceKey(vSourceKey) + ,count(vCount) + { + } + + bool operator<(const CacheItem &other) const + { + return count > other.count; + } +}; + +class CacheItemOrderer +{ +public: + bool operator()(const CacheItem* a, const CacheItem* b) const + { + return (*a) < (*b); + } +}; + +void serialize_cache( + std::priority_queue<CacheItem*, std::vector<CacheItem*>, CacheItemOrderer> &cache, + const std::string &path, float totalSourceCount); + +} + diff --git a/moses2/TranslationModel/ProbingPT/vocabid.cpp b/moses2/TranslationModel/ProbingPT/vocabid.cpp new file mode 100644 index 000000000..696373ee5 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/vocabid.cpp @@ -0,0 +1,59 @@ +#include <boost/foreach.hpp> +#include "vocabid.hh" +#include "StoreVocab.h" +#include "../../legacy/Util2.h" + +namespace Moses2 +{ + +void add_to_map(StoreVocab<uint64_t> &sourceVocab, + const StringPiece &textin) +{ + //Tokenize + util::TokenIter<util::SingleCharacter> itWord(textin, util::SingleCharacter(' ')); + + while (itWord) { + StringPiece word = *itWord; + + util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|')); + while (itFactor) { + StringPiece factor = *itFactor; + + sourceVocab.Insert(getHash(factor), factor.as_string()); + itFactor++; + } + itWord++; + } +} + +void serialize_map(const std::map<uint64_t, std::string> &karta, + const std::string &filename) +{ + std::ofstream os(filename.c_str()); + + std::map<uint64_t, std::string>::const_iterator iter; + for (iter = karta.begin(); iter != karta.end(); ++iter) { + os << iter->first << '\t' << iter->second << std::endl; + } + + os.close(); +} + +void read_map(std::map<uint64_t, std::string> &karta, const char* filename) +{ + std::ifstream is(filename); + + std::string line; + while (getline(is, line)) { + std::vector<std::string> toks = Tokenize(line, "\t"); + assert(toks.size() == 2); + uint64_t ind = Scan<uint64_t>(toks[1]); + karta[ind] = toks[0]; + } + + //Close the stream after we are done. + is.close(); +} + +} + diff --git a/moses2/TranslationModel/ProbingPT/vocabid.hh b/moses2/TranslationModel/ProbingPT/vocabid.hh new file mode 100644 index 000000000..55d99d453 --- /dev/null +++ b/moses2/TranslationModel/ProbingPT/vocabid.hh @@ -0,0 +1,29 @@ +//Serialization +#include <boost/serialization/serialization.hpp> +#include <boost/serialization/map.hpp> +#include <boost/archive/text_iarchive.hpp> +#include <boost/archive/text_oarchive.hpp> +#include <fstream> +#include <iostream> +#include <vector> + +#include <map> //Container +#include "hash.hh" //Hash of elements + +#include "util/string_piece.hh" //Tokenization and work with StringPiece +#include "util/tokenize_piece.hh" + +namespace Moses2 +{ +template<typename VOCABID> +class StoreVocab; + +void add_to_map(StoreVocab<uint64_t> &sourceVocab, + const StringPiece &textin); + +void serialize_map(const std::map<uint64_t, std::string> &karta, + const std::string &filename); + +void read_map(std::map<uint64_t, std::string> &karta, const char* filename); + +} diff --git a/moses2/TranslationModel/Transliteration.cpp b/moses2/TranslationModel/Transliteration.cpp new file mode 100644 index 000000000..f92348ee9 --- /dev/null +++ b/moses2/TranslationModel/Transliteration.cpp @@ -0,0 +1,229 @@ +/* + * Transliteration.cpp + * + * Created on: 28 Oct 2015 + * Author: hieu + */ +#include <boost/foreach.hpp> +#include "Transliteration.h" +#include "../System.h" +#include "../Scores.h" +#include "../InputType.h" +#include "../PhraseBased/Manager.h" +#include "../PhraseBased/TargetPhraseImpl.h" +#include "../PhraseBased/InputPath.h" +#include "../PhraseBased/TargetPhrases.h" +#include "../PhraseBased/Sentence.h" +#include "../SCFG/InputPath.h" +#include "../SCFG/TargetPhraseImpl.h" +#include "../SCFG/Manager.h" +#include "../SCFG/Sentence.h" +#include "../SCFG/ActiveChart.h" +#include "util/tempfile.hh" +#include "../legacy/Util2.h" + +using namespace std; + +namespace Moses2 +{ + +Transliteration::Transliteration(size_t startInd, const std::string &line) : + PhraseTable(startInd, line) +{ + ReadParameters(); + UTIL_THROW_IF2(m_mosesDir.empty() || + m_scriptDir.empty() || + m_externalDir.empty() || + m_inputLang.empty() || + m_outputLang.empty(), "Must specify all arguments"); +} + +Transliteration::~Transliteration() +{ + // TODO Auto-generated destructor stub +} + +void +Transliteration:: +SetParameter(const std::string& key, const std::string& value) +{ + if (key == "moses-dir") { + m_mosesDir = value; + } else if (key == "script-dir") { + m_scriptDir = value; + } else if (key == "external-dir") { + m_externalDir = value; + } else if (key == "input-lang") { + m_inputLang = value; + } else if (key == "output-lang") { + m_outputLang = value; + } else { + PhraseTable::SetParameter(key, value); + } +} + +void Transliteration::Lookup(const Manager &mgr, + InputPathsBase &inputPaths) const +{ + BOOST_FOREACH(InputPathBase *pathBase, inputPaths){ + InputPath *path = static_cast<InputPath*>(pathBase); + + if (SatisfyBackoff(mgr, *path)) { + const SubPhrase<Moses2::Word> &phrase = path->subPhrase; + + TargetPhrases *tps = Lookup(mgr, mgr.GetPool(), *path); + path->AddTargetPhrases(*this, tps); + } + } + +} + +TargetPhrases *Transliteration::Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const +{ + const SubPhrase<Moses2::Word> &sourcePhrase = inputPath.subPhrase; + size_t hash = sourcePhrase.hash(); + + // TRANSLITERATE + const util::temp_file inFile; + const util::temp_dir outDir; + + ofstream inStream(inFile.path().c_str()); + inStream << sourcePhrase.Debug(mgr.system) << endl; + inStream.close(); + + string cmd = m_scriptDir + "/Transliteration/prepare-transliteration-phrase-table.pl" + + " --transliteration-model-dir " + m_filePath + + " --moses-src-dir " + m_mosesDir + + " --external-bin-dir " + m_externalDir + + " --input-extension " + m_inputLang + + " --output-extension " + m_outputLang + + " --oov-file " + inFile.path() + + " --out-dir " + outDir.path(); + + int ret = system(cmd.c_str()); + UTIL_THROW_IF2(ret != 0, "Transliteration script error"); + + TargetPhrases *tps = NULL; + tps = new (pool.Allocate<TargetPhrases>()) TargetPhrases(pool, 1); + + vector<TargetPhraseImpl*> targetPhrases + = CreateTargetPhrases(mgr, pool, sourcePhrase, outDir.path()); + + vector<TargetPhraseImpl*>::const_iterator iter; + for (iter = targetPhrases.begin(); iter != targetPhrases.end(); ++iter) { + TargetPhraseImpl *tp = *iter; + tps->AddTargetPhrase(*tp); + } + mgr.system.featureFunctions.EvaluateAfterTablePruning(pool, *tps, sourcePhrase); + + inputPath.AddTargetPhrases(*this, tps); +} + +std::vector<TargetPhraseImpl*> Transliteration::CreateTargetPhrases( + const Manager &mgr, + MemPool &pool, + const SubPhrase<Moses2::Word> &sourcePhrase, + const std::string &outDir) const +{ + std::vector<TargetPhraseImpl*> ret; + + string outPath = outDir + "/out.txt"; + ifstream outStream(outPath.c_str()); + + string line; + while (getline(outStream, line)) { + vector<string> toks = Moses2::Tokenize(line, "\t"); + UTIL_THROW_IF2(toks.size() != 2, "Error in transliteration output file. Expecting word\tscore"); + + TargetPhraseImpl *tp = + new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, *this, mgr.system, 1); + Moses2::Word &word = (*tp)[0]; + word.CreateFromString(mgr.system.GetVocab(), mgr.system, toks[0]); + + float score = Scan<float>(toks[1]); + tp->GetScores().PlusEquals(mgr.system, *this, score); + + // score of all other ff when this rule is being loaded + mgr.system.featureFunctions.EvaluateInIsolation(pool, mgr.system, sourcePhrase, *tp); + + ret.push_back(tp); + } + + outStream.close(); + + return ret; + +} + + +void Transliteration::EvaluateInIsolation(const System &system, + const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const +{ + UTIL_THROW2("Not implemented"); +} + +// SCFG /////////////////////////////////////////////////////////////////////////////////////////// +void Transliteration::InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const +{ + UTIL_THROW2("Not implemented"); +} + +void Transliteration::Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const +{ + UTIL_THROW2("Not implemented"); +} + +void Transliteration::LookupUnary(MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const +{ + UTIL_THROW2("Not implemented"); +} + +void Transliteration::LookupNT( + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const +{ + UTIL_THROW2("Not implemented"); +} + +void Transliteration::LookupGivenWord( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const +{ + UTIL_THROW2("Not implemented"); +} + +void Transliteration::LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const +{ + UTIL_THROW2("Not implemented"); +} + +} + diff --git a/moses2/TranslationModel/Transliteration.h b/moses2/TranslationModel/Transliteration.h new file mode 100644 index 000000000..15f262ac8 --- /dev/null +++ b/moses2/TranslationModel/Transliteration.h @@ -0,0 +1,91 @@ +/* + * Transliteration.h + * + * Created on: 28 Oct 2015 + * Author: hieu + */ + +#pragma once + +#include "PhraseTable.h" + +namespace Moses2 +{ +class Sentence; +class InputPaths; +class Range; + +class Transliteration: public PhraseTable +{ +public: + Transliteration(size_t startInd, const std::string &line); + virtual ~Transliteration(); + + void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const; + virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const; + + virtual void + EvaluateInIsolation(const System &system, const Phrase<Moses2::Word> &source, + const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const; + + virtual void InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; + + void Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; + + void LookupUnary(MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; + +protected: + virtual void LookupNT( + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const; + + virtual void LookupGivenWord( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; + + virtual void LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; + + void SetParameter(const std::string& key, const std::string& value); + +protected: + std::string m_filePath; + std::string m_mosesDir, m_scriptDir, m_externalDir, m_inputLang, m_outputLang; + + std::vector<TargetPhraseImpl*> CreateTargetPhrases( + const Manager &mgr, + MemPool &pool, + const SubPhrase<Moses2::Word> &sourcePhrase, + const std::string &outDir) const; + +}; + +} + diff --git a/moses2/TranslationModel/UnknownWordPenalty.cpp b/moses2/TranslationModel/UnknownWordPenalty.cpp new file mode 100644 index 000000000..d786b2cff --- /dev/null +++ b/moses2/TranslationModel/UnknownWordPenalty.cpp @@ -0,0 +1,285 @@ +/* + * UnknownWordPenalty.cpp + * + * Created on: 28 Oct 2015 + * Author: hieu + */ +#include <boost/foreach.hpp> +#include "UnknownWordPenalty.h" +#include "../System.h" +#include "../Scores.h" +#include "../InputType.h" +#include "../PhraseBased/Manager.h" +#include "../PhraseBased/TargetPhraseImpl.h" +#include "../PhraseBased/InputPath.h" +#include "../PhraseBased/TargetPhrases.h" +#include "../PhraseBased/Sentence.h" +#include "../SCFG/InputPath.h" +#include "../SCFG/TargetPhraseImpl.h" +#include "../SCFG/Manager.h" +#include "../SCFG/Sentence.h" +#include "../SCFG/ActiveChart.h" + +using namespace std; + +namespace Moses2 +{ + +UnknownWordPenalty::UnknownWordPenalty(size_t startInd, const std::string &line) +:PhraseTable(startInd, line) +,m_drop(false) +{ + m_tuneable = false; + ReadParameters(); +} + +UnknownWordPenalty::~UnknownWordPenalty() +{ + // TODO Auto-generated destructor stub +} + +void UnknownWordPenalty::SetParameter(const std::string& key, const std::string& value) +{ + if (key == "drop") { + m_drop = Scan<bool>(value); + } + else if (key == "prefix") { + m_prefix = value; + } + else if (key == "suffix") { + m_suffix = value; + } + else { + PhraseTable::SetParameter(key, value); + } +} + +void UnknownWordPenalty::ProcessXML( + const Manager &mgr, + MemPool &pool, + const Sentence &sentence, + InputPaths &inputPaths) const +{ + const Vector<const InputType::XMLOption*> &xmlOptions = sentence.GetXMLOptions(); + BOOST_FOREACH(const InputType::XMLOption *xmlOption, xmlOptions) { + TargetPhraseImpl *target = TargetPhraseImpl::CreateFromString(pool, *this, mgr.system, xmlOption->GetTranslation()); + + if (xmlOption->prob) { + Scores &scores = target->GetScores(); + scores.PlusEquals(mgr.system, *this, Moses2::TransformScore(xmlOption->prob)); + } + + InputPath *path = inputPaths.GetMatrix().GetValue(xmlOption->startPos, xmlOption->phraseSize - 1); + const SubPhrase<Moses2::Word> &source = path->subPhrase; + + mgr.system.featureFunctions.EvaluateInIsolation(pool, mgr.system, source, *target); + + TargetPhrases *tps = new (pool.Allocate<TargetPhrases>()) TargetPhrases(pool, 1); + + tps->AddTargetPhrase(*target); + mgr.system.featureFunctions.EvaluateAfterTablePruning(pool, *tps, source); + + path->AddTargetPhrases(*this, tps); + } +} + +void UnknownWordPenalty::Lookup(const Manager &mgr, + InputPathsBase &inputPaths) const +{ + BOOST_FOREACH(InputPathBase *pathBase, inputPaths){ + InputPath *path = static_cast<InputPath*>(pathBase); + + if (SatisfyBackoff(mgr, *path)) { + const SubPhrase<Moses2::Word> &phrase = path->subPhrase; + + TargetPhrases *tps = Lookup(mgr, mgr.GetPool(), *path); + path->AddTargetPhrases(*this, tps); + } + } + +} + +TargetPhrases *UnknownWordPenalty::Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const +{ + const System &system = mgr.system; + TargetPhrases *tps = NULL; + + // any other pt translate this? + size_t numPt = mgr.system.mappings.size(); + const TargetPhrases **allTPS = + static_cast<InputPath&>(inputPath).targetPhrases; + for (size_t i = 0; i < numPt; ++i) { + const TargetPhrases *otherTps = allTPS[i]; + + if (otherTps && otherTps->GetSize()) { + return tps; + } + } + + const SubPhrase<Moses2::Word> &source = inputPath.subPhrase; + const Moses2::Word &sourceWord = source[0]; + const Factor *factor = sourceWord[0]; + + tps = new (pool.Allocate<TargetPhrases>()) TargetPhrases(pool, 1); + + size_t numWords = m_drop ? 0 : 1; + + TargetPhraseImpl *target = + new (pool.Allocate<TargetPhraseImpl>()) TargetPhraseImpl(pool, *this, + system, numWords); + + if (!m_drop) { + Moses2::Word &word = (*target)[0]; + + if (m_prefix.empty() && m_suffix.empty()) { + word[0] = factor; + } + else { + stringstream strm; + if (!m_prefix.empty()) { + strm << m_prefix; + } + strm << factor->GetString(); + if (!m_suffix.empty()) { + strm << m_suffix; + } + + FactorCollection &fc = system.GetVocab(); + const Factor *targetFactor = fc.AddFactor(strm.str(), system, false); + word[0] = targetFactor; + } + } + + Scores &scores = target->GetScores(); + scores.PlusEquals(mgr.system, *this, -100); + + MemPool &memPool = mgr.GetPool(); + system.featureFunctions.EvaluateInIsolation(memPool, system, source, *target); + + tps->AddTargetPhrase(*target); + system.featureFunctions.EvaluateAfterTablePruning(memPool, *tps, source); + + return tps; +} + +void UnknownWordPenalty::EvaluateInIsolation(const System &system, + const Phrase<Moses2::Word> &source, const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const +{ + +} + +// SCFG /////////////////////////////////////////////////////////////////////////////////////////// +void UnknownWordPenalty::InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const +{ +} + +void UnknownWordPenalty::Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const +{ + const System &system = mgr.system; + + size_t numWords = path.range.GetNumWordsCovered(); + if (numWords > 1) { + // only create 1 word phrases + return; + } + + if (path.GetNumRules()) { + // only create rules if no other rules + return; + } + + // don't do 1st if 1st word + if (path.range.GetStartPos() == 0) { + return; + } + + // don't do 1st if last word + const SCFG::Sentence &sentence = static_cast<const SCFG::Sentence&>(mgr.GetInput()); + if (path.range.GetStartPos() + 1 == sentence.GetSize()) { + return; + } + + // terminal + const SCFG::Word &lastWord = path.subPhrase.Back(); + //cerr << "UnknownWordPenalty lastWord=" << lastWord << endl; + + const Factor *factor = lastWord[0]; + SCFG::TargetPhraseImpl *tp = new (pool.Allocate<SCFG::TargetPhraseImpl>()) SCFG::TargetPhraseImpl(pool, *this, system, 1); + SCFG::Word &word = (*tp)[0]; + word.CreateFromString(system.GetVocab(), system, factor->GetString().as_string()); + + tp->lhs.CreateFromString(system.GetVocab(), system, "[X]"); + + size_t endPos = path.range.GetEndPos(); + const SCFG::InputPath &subPhrasePath = *mgr.GetInputPaths().GetMatrix().GetValue(endPos, 1); + + SCFG::ActiveChartEntry *chartEntry = new (pool.Allocate<SCFG::ActiveChartEntry>()) SCFG::ActiveChartEntry(pool); + chartEntry->AddSymbolBindElement(subPhrasePath.range, lastWord, NULL, *this); + path.AddActiveChartEntry(GetPtInd(), chartEntry); + + Scores &scores = tp->GetScores(); + scores.PlusEquals(mgr.system, *this, -100); + + MemPool &memPool = mgr.GetPool(); + const SubPhrase<SCFG::Word> &source = path.subPhrase; + system.featureFunctions.EvaluateInIsolation(memPool, system, source, *tp); + + SCFG::TargetPhrases *tps = new (pool.Allocate<SCFG::TargetPhrases>()) SCFG::TargetPhrases(pool); + tps->AddTargetPhrase(*tp); + + path.AddTargetPhrasesToPath(pool, mgr.system, *this, *tps, chartEntry->GetSymbolBind()); +} + +void UnknownWordPenalty::LookupUnary(MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const +{ +} + +void UnknownWordPenalty::LookupNT( + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const +{ + UTIL_THROW2("Not implemented"); +} + +void UnknownWordPenalty::LookupGivenWord( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const +{ + UTIL_THROW2("Not implemented"); +} + +void UnknownWordPenalty::LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const +{ + UTIL_THROW2("Not implemented"); +} + +} + diff --git a/moses2/TranslationModel/UnknownWordPenalty.h b/moses2/TranslationModel/UnknownWordPenalty.h new file mode 100644 index 000000000..52c235a36 --- /dev/null +++ b/moses2/TranslationModel/UnknownWordPenalty.h @@ -0,0 +1,89 @@ +/* + * UnknownWordPenalty.h + * + * Created on: 28 Oct 2015 + * Author: hieu + */ + +#pragma once + +#include "PhraseTable.h" + +namespace Moses2 +{ +class Sentence; +class InputPaths; +class Range; + +class UnknownWordPenalty: public PhraseTable +{ +public: + UnknownWordPenalty(size_t startInd, const std::string &line); + virtual ~UnknownWordPenalty(); + + virtual void SetParameter(const std::string& key, const std::string& value); + + void Lookup(const Manager &mgr, InputPathsBase &inputPaths) const; + virtual TargetPhrases *Lookup(const Manager &mgr, MemPool &pool, + InputPath &inputPath) const; + + void ProcessXML( + const Manager &mgr, + MemPool &pool, + const Sentence &sentence, + InputPaths &inputPaths) const; + + virtual void + EvaluateInIsolation(const System &system, const Phrase<Moses2::Word> &source, + const TargetPhraseImpl &targetPhrase, Scores &scores, + SCORE &estimatedScore) const; + + virtual void InitActiveChart( + MemPool &pool, + const SCFG::Manager &mgr, + SCFG::InputPath &path) const; + + void Lookup(MemPool &pool, + const SCFG::Manager &mgr, + size_t maxChartSpan, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; + + void LookupUnary(MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::Stacks &stacks, + SCFG::InputPath &path) const; + +protected: + virtual void LookupNT( + MemPool &pool, + const SCFG::Manager &mgr, + const Moses2::Range &subPhraseRange, + const SCFG::InputPath &prevPath, + const SCFG::Stacks &stacks, + SCFG::InputPath &outPath) const; + + virtual void LookupGivenWord( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::InputPath &prevPath, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; + + virtual void LookupGivenNode( + MemPool &pool, + const SCFG::Manager &mgr, + const SCFG::ActiveChartEntry &prevEntry, + const SCFG::Word &wordSought, + const Moses2::Hypotheses *hypos, + const Moses2::Range &subPhraseRange, + SCFG::InputPath &outPath) const; +protected: + bool m_drop; + std::string m_prefix, m_suffix; +}; + +} + |