diff options
-rw-r--r-- | OnDiskPt/OnDiskWrapper.cpp | 14 | ||||
-rw-r--r-- | OnDiskPt/OnDiskWrapper.h | 4 | ||||
-rw-r--r-- | OnDiskPt/PhraseNode.cpp | 42 | ||||
-rw-r--r-- | OnDiskPt/PhraseNode.h | 18 | ||||
-rw-r--r-- | OnDiskPt/TargetPhrase.cpp | 94 | ||||
-rw-r--r-- | OnDiskPt/TargetPhrase.h | 16 | ||||
-rw-r--r-- | OnDiskPt/TargetPhraseCollection.cpp | 28 | ||||
-rw-r--r-- | OnDiskPt/TargetPhraseCollection.h | 6 | ||||
-rw-r--r-- | OnDiskPt/Vocab.cpp | 10 | ||||
-rw-r--r-- | OnDiskPt/Vocab.h | 10 | ||||
-rw-r--r-- | OnDiskPt/Word.cpp | 12 | ||||
-rw-r--r-- | OnDiskPt/Word.h | 4 | ||||
-rw-r--r-- | moses/File.h | 16 | ||||
-rw-r--r-- | moses/PrefixTreeMap.cpp | 12 | ||||
-rw-r--r-- | moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp | 6 | ||||
-rw-r--r-- | moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h | 2 | ||||
-rw-r--r-- | moses/TypeDef.h | 9 |
17 files changed, 148 insertions, 155 deletions
diff --git a/OnDiskPt/OnDiskWrapper.cpp b/OnDiskPt/OnDiskWrapper.cpp index 91935e965..3bfb7d2b1 100644 --- a/OnDiskPt/OnDiskWrapper.cpp +++ b/OnDiskPt/OnDiskWrapper.cpp @@ -51,7 +51,7 @@ void OnDiskWrapper::BeginLoad(const std::string &filePath) if (!m_vocab.Load(*this)) UTIL_THROW(util::FileOpenException, "Couldn't load vocab"); - UINT64 rootFilePos = GetMisc("RootNodeOffset"); + uint64_t rootFilePos = GetMisc("RootNodeOffset"); m_rootSourceNode = new PhraseNode(rootFilePos, *this); } @@ -102,7 +102,7 @@ bool OnDiskWrapper::LoadMisc() const string &key = tokens[0]; - m_miscInfo[key] = Moses::Scan<UINT64>(tokens[1]); + m_miscInfo[key] = Moses::Scan<uint64_t>(tokens[1]); } return true; @@ -199,17 +199,17 @@ void OnDiskWrapper::SaveMisc() size_t OnDiskWrapper::GetSourceWordSize() const { - return sizeof(UINT64) + sizeof(char); + return sizeof(uint64_t) + sizeof(char); } size_t OnDiskWrapper::GetTargetWordSize() const { - return sizeof(UINT64) + sizeof(char); + return sizeof(uint64_t) + sizeof(char); } -UINT64 OnDiskWrapper::GetMisc(const std::string &key) const +uint64_t OnDiskWrapper::GetMisc(const std::string &key) const { - std::map<std::string, UINT64>::const_iterator iter; + std::map<std::string, uint64_t>::const_iterator iter; iter = m_miscInfo.find(key); UTIL_THROW_IF2(iter == m_miscInfo.end() , "Couldn't find value for key " << key @@ -243,7 +243,7 @@ Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &fact } // for (size_t factorType bool found; - UINT64 vocabId = m_vocab.GetVocabId(strme.str(), found); + uint64_t vocabId = m_vocab.GetVocabId(strme.str(), found); if (!found) { // factor not in phrase table -> phrse definately not in. exit delete newWord; diff --git a/OnDiskPt/OnDiskWrapper.h b/OnDiskPt/OnDiskWrapper.h index 5b7cdfe01..bf398506c 100644 --- a/OnDiskPt/OnDiskWrapper.h +++ b/OnDiskPt/OnDiskWrapper.h @@ -43,7 +43,7 @@ protected: size_t m_defaultNodeSize; PhraseNode *m_rootSourceNode; - std::map<std::string, UINT64> m_miscInfo; + std::map<std::string, uint64_t> m_miscInfo; void SaveMisc(); bool OpenForLoad(const std::string &filePath); @@ -105,7 +105,7 @@ public: return *m_rootSourceNode; } - UINT64 GetMisc(const std::string &key) const; + uint64_t GetMisc(const std::string &key) const; Word *ConvertFromMoses(const std::vector<Moses::FactorType> &factorsVec , const Moses::Word &origWord) const; diff --git a/OnDiskPt/PhraseNode.cpp b/OnDiskPt/PhraseNode.cpp index 01cfda847..8e50147b2 100644 --- a/OnDiskPt/PhraseNode.cpp +++ b/OnDiskPt/PhraseNode.cpp @@ -31,8 +31,8 @@ namespace OnDiskPt size_t PhraseNode::GetNodeSize(size_t numChildren, size_t wordSize, size_t countSize) { - size_t ret = sizeof(UINT64) * 2 // num children, value - + (wordSize + sizeof(UINT64)) * numChildren // word + ptr to next source node + size_t ret = sizeof(uint64_t) * 2 // num children, value + + (wordSize + sizeof(uint64_t)) * numChildren // word + ptr to next source node + sizeof(float) * countSize; // count info return ret; } @@ -45,7 +45,7 @@ PhraseNode::PhraseNode() { } -PhraseNode::PhraseNode(UINT64 filePos, OnDiskWrapper &onDiskWrapper) +PhraseNode::PhraseNode(uint64_t filePos, OnDiskWrapper &onDiskWrapper) :m_counts(onDiskWrapper.GetNumCounts()) { // load saved node @@ -55,26 +55,26 @@ PhraseNode::PhraseNode(UINT64 filePos, OnDiskWrapper &onDiskWrapper) std::fstream &file = onDiskWrapper.GetFileSource(); file.seekg(filePos); - assert(filePos == (UINT64)file.tellg()); + assert(filePos == (uint64_t)file.tellg()); - file.read((char*) &m_numChildrenLoad, sizeof(UINT64)); + file.read((char*) &m_numChildrenLoad, sizeof(uint64_t)); size_t memAlloc = GetNodeSize(m_numChildrenLoad, onDiskWrapper.GetSourceWordSize(), countSize); m_memLoad = (char*) malloc(memAlloc); // go to start of node again file.seekg(filePos); - assert(filePos == (UINT64)file.tellg()); + assert(filePos == (uint64_t)file.tellg()); // read everything into memory file.read(m_memLoad, memAlloc); - assert(filePos + memAlloc == (UINT64)file.tellg()); + assert(filePos + memAlloc == (uint64_t)file.tellg()); // get value - m_value = ((UINT64*)m_memLoad)[1]; + m_value = ((uint64_t*)m_memLoad)[1]; // get counts - float *memFloat = (float*) (m_memLoad + sizeof(UINT64) * 2); + float *memFloat = (float*) (m_memLoad + sizeof(uint64_t) * 2); assert(countSize == 1); m_counts[0] = memFloat[0]; @@ -108,10 +108,10 @@ void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimi //memset(mem, 0xfe, memAlloc); size_t memUsed = 0; - UINT64 *memArray = (UINT64*) mem; + uint64_t *memArray = (uint64_t*) mem; memArray[0] = GetSize(); // num of children memArray[1] = m_value; // file pos of corresponding target phrases - memUsed += 2 * sizeof(UINT64); + memUsed += 2 * sizeof(uint64_t); // count info float *memFloat = (float*) (mem + memUsed); @@ -133,9 +133,9 @@ void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimi size_t wordMemUsed = childWord.WriteToMemory(currMem); memUsed += wordMemUsed; - UINT64 *memArray = (UINT64*) (mem + memUsed); + uint64_t *memArray = (uint64_t*) (mem + memUsed); memArray[0] = childNode.GetFilePos(); - memUsed += sizeof(UINT64); + memUsed += sizeof(uint64_t); } @@ -148,7 +148,7 @@ void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimi file.seekp(0, ios::end); file.write(mem, memUsed); - UINT64 endPos = file.tellp(); + uint64_t endPos = file.tellp(); assert(m_filePos + memUsed == endPos); free(mem); @@ -206,7 +206,7 @@ const PhraseNode *PhraseNode::GetChild(const Word &wordSought, OnDiskWrapper &on x = (l + r) / 2; Word wordFound; - UINT64 childFilePos; + uint64_t childFilePos; GetChild(wordFound, childFilePos, x, onDiskWrapper); if (wordSought == wordFound) { @@ -222,14 +222,14 @@ const PhraseNode *PhraseNode::GetChild(const Word &wordSought, OnDiskWrapper &on return ret; } -void PhraseNode::GetChild(Word &wordFound, UINT64 &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const +void PhraseNode::GetChild(Word &wordFound, uint64_t &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const { size_t wordSize = onDiskWrapper.GetSourceWordSize(); - size_t childSize = wordSize + sizeof(UINT64); + size_t childSize = wordSize + sizeof(uint64_t); char *currMem = m_memLoad - + sizeof(UINT64) * 2 // size & file pos of target phrase coll + + sizeof(uint64_t) * 2 // size & file pos of target phrase coll + sizeof(float) * onDiskWrapper.GetNumCounts() // count info + childSize * ind; @@ -237,15 +237,15 @@ void PhraseNode::GetChild(Word &wordFound, UINT64 &childFilePos, size_t ind, OnD assert(memRead == childSize); } -size_t PhraseNode::ReadChild(Word &wordFound, UINT64 &childFilePos, const char *mem) const +size_t PhraseNode::ReadChild(Word &wordFound, uint64_t &childFilePos, const char *mem) const { size_t memRead = wordFound.ReadFromMemory(mem); const char *currMem = mem + memRead; - UINT64 *memArray = (UINT64*) (currMem); + uint64_t *memArray = (uint64_t*) (currMem); childFilePos = memArray[0]; - memRead += sizeof(UINT64); + memRead += sizeof(uint64_t); return memRead; } diff --git a/OnDiskPt/PhraseNode.h b/OnDiskPt/PhraseNode.h index 6b629a401..901852952 100644 --- a/OnDiskPt/PhraseNode.h +++ b/OnDiskPt/PhraseNode.h @@ -36,7 +36,7 @@ class PhraseNode { friend std::ostream& operator<<(std::ostream&, const PhraseNode&); protected: - UINT64 m_filePos, m_value; + uint64_t m_filePos, m_value; typedef std::map<Word, PhraseNode> ChildColl; ChildColl m_children; @@ -48,35 +48,35 @@ protected: TargetPhraseCollection m_targetPhraseColl; char *m_memLoad, *m_memLoadLast; - UINT64 m_numChildrenLoad; + uint64_t m_numChildrenLoad; void AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase , TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper , size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort); - size_t ReadChild(Word &wordFound, UINT64 &childFilePos, const char *mem) const; - void GetChild(Word &wordFound, UINT64 &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const; + size_t ReadChild(Word &wordFound, uint64_t &childFilePos, const char *mem) const; + void GetChild(Word &wordFound, uint64_t &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const; public: static size_t GetNodeSize(size_t numChildren, size_t wordSize, size_t countSize); PhraseNode(); // unsaved node - PhraseNode(UINT64 filePos, OnDiskWrapper &onDiskWrapper); // load saved node + PhraseNode(uint64_t filePos, OnDiskWrapper &onDiskWrapper); // load saved node ~PhraseNode(); - void Add(const Word &word, UINT64 nextFilePos, size_t wordSize); + void Add(const Word &word, uint64_t nextFilePos, size_t wordSize); void Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimit); void AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase , OnDiskWrapper &onDiskWrapper, size_t tableLimit , const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort); - UINT64 GetFilePos() const { + uint64_t GetFilePos() const { return m_filePos; } - UINT64 GetValue() const { + uint64_t GetValue() const { return m_value; } - void SetValue(UINT64 value) { + void SetValue(uint64_t value) { m_value = value; } size_t GetSize() const { diff --git a/OnDiskPt/TargetPhrase.cpp b/OnDiskPt/TargetPhrase.cpp index 5d1dc8e5b..f0501a937 100644 --- a/OnDiskPt/TargetPhrase.cpp +++ b/OnDiskPt/TargetPhrase.cpp @@ -103,17 +103,17 @@ char *TargetPhrase::WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) size_t spSize = sp->GetSize(); size_t sourceWordSize = onDiskWrapper.GetSourceWordSize(); - size_t memNeeded = sizeof(UINT64) // num of words + size_t memNeeded = sizeof(uint64_t) // num of words + targetWordSize * phraseSize // actual words. lhs as last words - + sizeof(UINT64) // num source words + + sizeof(uint64_t) // num source words + sourceWordSize * spSize; // actual source words memUsed = 0; - UINT64 *mem = (UINT64*) malloc(memNeeded); + uint64_t *mem = (uint64_t*) malloc(memNeeded); // write size mem[0] = phraseSize; - memUsed += sizeof(UINT64); + memUsed += sizeof(uint64_t); // write each word for (size_t pos = 0; pos < phraseSize; ++pos) { @@ -124,9 +124,9 @@ char *TargetPhrase::WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) // write size of source phrase and all source words char *currPtr = (char*)mem + memUsed; - UINT64 *memTmp = (UINT64*) currPtr; + uint64_t *memTmp = (uint64_t*) currPtr; memTmp[0] = spSize; - memUsed += sizeof(UINT64); + memUsed += sizeof(uint64_t); for (size_t pos = 0; pos < spSize; ++pos) { const Word &word = sp->GetWord(pos); char *currPtr = (char*)mem + memUsed; @@ -145,13 +145,13 @@ void TargetPhrase::Save(OnDiskWrapper &onDiskWrapper) std::fstream &file = onDiskWrapper.GetFileTargetInd(); - UINT64 startPos = file.tellp(); + uint64_t startPos = file.tellp(); file.seekp(0, ios::end); file.write(mem, memUsed); #ifndef NDEBUG - UINT64 endPos = file.tellp(); + uint64_t endPos = file.tellp(); assert(startPos + memUsed == endPos); #endif @@ -167,11 +167,11 @@ char *TargetPhrase::WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t size_t sparseFeatureSize = m_sparseFeatures.size(); size_t propSize = m_property.size(); - size_t memNeeded = sizeof(UINT64) // file pos (phrase id) - + sizeof(UINT64) + 2 * sizeof(UINT64) * numAlign // align + size_t memNeeded = sizeof(uint64_t) // file pos (phrase id) + + sizeof(uint64_t) + 2 * sizeof(uint64_t) * numAlign // align + sizeof(float) * numScores // scores - + sizeof(UINT64) + sparseFeatureSize // sparse features string - + sizeof(UINT64) + propSize; // property string + + sizeof(uint64_t) + sparseFeatureSize // sparse features string + + sizeof(uint64_t) + propSize; // property string char *mem = (char*) malloc(memNeeded); //memset(mem, 0, memNeeded); @@ -179,8 +179,8 @@ char *TargetPhrase::WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t memUsed = 0; // phrase id - memcpy(mem, &m_filePos, sizeof(UINT64)); - memUsed += sizeof(UINT64); + memcpy(mem, &m_filePos, sizeof(uint64_t)); + memUsed += sizeof(uint64_t); // align size_t tmp = WriteAlignToMemory(mem + memUsed); @@ -203,11 +203,11 @@ char *TargetPhrase::WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t size_t TargetPhrase::WriteStringToMemory(char *mem, const std::string &str) const { size_t memUsed = 0; - UINT64 *memTmp = (UINT64*) mem; + uint64_t *memTmp = (uint64_t*) mem; size_t strSize = str.size(); memTmp[0] = strSize; - memUsed += sizeof(UINT64); + memUsed += sizeof(uint64_t); const char *charStr = str.c_str(); memcpy(mem + memUsed, charStr, strSize); @@ -221,7 +221,7 @@ size_t TargetPhrase::WriteAlignToMemory(char *mem) const size_t memUsed = 0; // num of alignments - UINT64 numAlign = m_align.size(); + uint64_t numAlign = m_align.size(); memcpy(mem, &numAlign, sizeof(numAlign)); memUsed += sizeof(numAlign); @@ -319,20 +319,20 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto return ret; } -UINT64 TargetPhrase::ReadOtherInfoFromFile(UINT64 filePos, std::fstream &fileTPColl) +uint64_t TargetPhrase::ReadOtherInfoFromFile(uint64_t filePos, std::fstream &fileTPColl) { - assert(filePos == (UINT64)fileTPColl.tellg()); + assert(filePos == (uint64_t)fileTPColl.tellg()); - UINT64 memUsed = 0; - fileTPColl.read((char*) &m_filePos, sizeof(UINT64)); - memUsed += sizeof(UINT64); + uint64_t memUsed = 0; + fileTPColl.read((char*) &m_filePos, sizeof(uint64_t)); + memUsed += sizeof(uint64_t); assert(m_filePos != 0); memUsed += ReadAlignFromFile(fileTPColl); - assert((memUsed + filePos) == (UINT64)fileTPColl.tellg()); + assert((memUsed + filePos) == (uint64_t)fileTPColl.tellg()); memUsed += ReadScoresFromFile(fileTPColl); - assert((memUsed + filePos) == (UINT64)fileTPColl.tellg()); + assert((memUsed + filePos) == (uint64_t)fileTPColl.tellg()); // sparse features memUsed += ReadStringFromFile(fileTPColl, m_sparseFeatures); @@ -343,13 +343,13 @@ UINT64 TargetPhrase::ReadOtherInfoFromFile(UINT64 filePos, std::fstream &fileTPC return memUsed; } -UINT64 TargetPhrase::ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr) +uint64_t TargetPhrase::ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr) { - UINT64 bytesRead = 0; + uint64_t bytesRead = 0; - UINT64 strSize; - fileTPColl.read((char*) &strSize, sizeof(UINT64)); - bytesRead += sizeof(UINT64); + uint64_t strSize; + fileTPColl.read((char*) &strSize, sizeof(uint64_t)); + bytesRead += sizeof(uint64_t); if (strSize) { char *mem = (char*) malloc(strSize + 1); @@ -364,15 +364,15 @@ UINT64 TargetPhrase::ReadStringFromFile(std::fstream &fileTPColl, std::string &o return bytesRead; } -UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP) +uint64_t TargetPhrase::ReadFromFile(std::fstream &fileTP) { - UINT64 bytesRead = 0; + uint64_t bytesRead = 0; fileTP.seekg(m_filePos); - UINT64 numWords; - fileTP.read((char*) &numWords, sizeof(UINT64)); - bytesRead += sizeof(UINT64); + uint64_t numWords; + fileTP.read((char*) &numWords, sizeof(uint64_t)); + bytesRead += sizeof(uint64_t); for (size_t ind = 0; ind < numWords; ++ind) { WordPtr word(new Word()); @@ -381,9 +381,9 @@ UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP) } // read source words - UINT64 numSourceWords; - fileTP.read((char*) &numSourceWords, sizeof(UINT64)); - bytesRead += sizeof(UINT64); + uint64_t numSourceWords; + fileTP.read((char*) &numSourceWords, sizeof(uint64_t)); + bytesRead += sizeof(uint64_t); PhrasePtr sp(new SourcePhrase()); for (size_t ind = 0; ind < numSourceWords; ++ind) { @@ -396,31 +396,31 @@ UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP) return bytesRead; } -UINT64 TargetPhrase::ReadAlignFromFile(std::fstream &fileTPColl) +uint64_t TargetPhrase::ReadAlignFromFile(std::fstream &fileTPColl) { - UINT64 bytesRead = 0; + uint64_t bytesRead = 0; - UINT64 numAlign; - fileTPColl.read((char*) &numAlign, sizeof(UINT64)); - bytesRead += sizeof(UINT64); + uint64_t numAlign; + fileTPColl.read((char*) &numAlign, sizeof(uint64_t)); + bytesRead += sizeof(uint64_t); for (size_t ind = 0; ind < numAlign; ++ind) { AlignPair alignPair; - fileTPColl.read((char*) &alignPair.first, sizeof(UINT64)); - fileTPColl.read((char*) &alignPair.second, sizeof(UINT64)); + fileTPColl.read((char*) &alignPair.first, sizeof(uint64_t)); + fileTPColl.read((char*) &alignPair.second, sizeof(uint64_t)); m_align.push_back(alignPair); - bytesRead += sizeof(UINT64) * 2; + bytesRead += sizeof(uint64_t) * 2; } return bytesRead; } -UINT64 TargetPhrase::ReadScoresFromFile(std::fstream &fileTPColl) +uint64_t TargetPhrase::ReadScoresFromFile(std::fstream &fileTPColl) { UTIL_THROW_IF2(m_scores.size() == 0, "Translation rules must must have some scores"); - UINT64 bytesRead = 0; + uint64_t bytesRead = 0; for (size_t ind = 0; ind < m_scores.size(); ++ind) { fileTPColl.read((char*) &m_scores[ind], sizeof(float)); diff --git a/OnDiskPt/TargetPhrase.h b/OnDiskPt/TargetPhrase.h index efd91a4ae..b41b788d5 100644 --- a/OnDiskPt/TargetPhrase.h +++ b/OnDiskPt/TargetPhrase.h @@ -36,7 +36,7 @@ class Phrase; namespace OnDiskPt { -typedef std::pair<UINT64, UINT64> AlignPair; +typedef std::pair<uint64_t, uint64_t> AlignPair; typedef std::vector<AlignPair> AlignType; class Vocab; @@ -53,15 +53,15 @@ protected: std::string m_sparseFeatures, m_property; std::vector<float> m_scores; - UINT64 m_filePos; + uint64_t m_filePos; size_t WriteAlignToMemory(char *mem) const; size_t WriteScoresToMemory(char *mem) const; size_t WriteStringToMemory(char *mem, const std::string &str) const; - UINT64 ReadAlignFromFile(std::fstream &fileTPColl); - UINT64 ReadScoresFromFile(std::fstream &fileTPColl); - UINT64 ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr); + uint64_t ReadAlignFromFile(std::fstream &fileTPColl); + uint64_t ReadScoresFromFile(std::fstream &fileTPColl); + uint64_t ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr); public: TargetPhrase() { @@ -95,7 +95,7 @@ public: char *WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) const; void Save(OnDiskWrapper &onDiskWrapper); - UINT64 GetFilePos() const { + uint64_t GetFilePos() const { return m_filePos; } float GetScore(size_t ind) const { @@ -108,8 +108,8 @@ public: , const Moses::PhraseDictionary &phraseDict , const std::vector<float> &weightT , bool isSyntax) const; - UINT64 ReadOtherInfoFromFile(UINT64 filePos, std::fstream &fileTPColl); - UINT64 ReadFromFile(std::fstream &fileTP); + uint64_t ReadOtherInfoFromFile(uint64_t filePos, std::fstream &fileTPColl); + uint64_t ReadFromFile(std::fstream &fileTP); virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const; diff --git a/OnDiskPt/TargetPhraseCollection.cpp b/OnDiskPt/TargetPhraseCollection.cpp index 7f6b5016f..73ad2540c 100644 --- a/OnDiskPt/TargetPhraseCollection.cpp +++ b/OnDiskPt/TargetPhraseCollection.cpp @@ -71,12 +71,12 @@ void TargetPhraseCollection::Save(OnDiskWrapper &onDiskWrapper) { std::fstream &file = onDiskWrapper.GetFileTargetColl(); - size_t memUsed = sizeof(UINT64); + size_t memUsed = sizeof(uint64_t); char *mem = (char*) malloc(memUsed); // size of coll - UINT64 numPhrases = GetSize(); - ((UINT64*)mem)[0] = numPhrases; + uint64_t numPhrases = GetSize(); + ((uint64_t*)mem)[0] = numPhrases; // MAIN LOOP CollType::iterator iter; @@ -98,16 +98,16 @@ void TargetPhraseCollection::Save(OnDiskWrapper &onDiskWrapper) } // total number of bytes - //((UINT64*)mem)[0] = (UINT64) memUsed; + //((uint64_t*)mem)[0] = (uint64_t) memUsed; - UINT64 startPos = file.tellp(); + uint64_t startPos = file.tellp(); file.seekp(0, ios::end); file.write((char*) mem, memUsed); free(mem); #ifndef NDEBUG - UINT64 endPos = file.tellp(); + uint64_t endPos = file.tellp(); assert(startPos + memUsed == endPos); #endif m_filePos = startPos; @@ -148,7 +148,7 @@ Moses::TargetPhraseCollection *TargetPhraseCollection::ConvertToMoses(const std: } -void TargetPhraseCollection::ReadFromFile(size_t tableLimit, UINT64 filePos, OnDiskWrapper &onDiskWrapper) +void TargetPhraseCollection::ReadFromFile(size_t tableLimit, uint64_t filePos, OnDiskWrapper &onDiskWrapper) { fstream &fileTPColl = onDiskWrapper.GetFileTargetColl(); fstream &fileTP = onDiskWrapper.GetFileTargetInd(); @@ -156,23 +156,23 @@ void TargetPhraseCollection::ReadFromFile(size_t tableLimit, UINT64 filePos, OnD size_t numScores = onDiskWrapper.GetNumScores(); - UINT64 numPhrases; + uint64_t numPhrases; - UINT64 currFilePos = filePos; + uint64_t currFilePos = filePos; fileTPColl.seekg(filePos); - fileTPColl.read((char*) &numPhrases, sizeof(UINT64)); + fileTPColl.read((char*) &numPhrases, sizeof(uint64_t)); // table limit if (tableLimit) { - numPhrases = std::min(numPhrases, (UINT64) tableLimit); + numPhrases = std::min(numPhrases, (uint64_t) tableLimit); } - currFilePos += sizeof(UINT64); + currFilePos += sizeof(uint64_t); for (size_t ind = 0; ind < numPhrases; ++ind) { TargetPhrase *tp = new TargetPhrase(numScores); - UINT64 sizeOtherInfo = tp->ReadOtherInfoFromFile(currFilePos, fileTPColl); + uint64_t sizeOtherInfo = tp->ReadOtherInfoFromFile(currFilePos, fileTPColl); tp->ReadFromFile(fileTP); currFilePos += sizeOtherInfo; @@ -181,7 +181,7 @@ void TargetPhraseCollection::ReadFromFile(size_t tableLimit, UINT64 filePos, OnD } } -UINT64 TargetPhraseCollection::GetFilePos() const +uint64_t TargetPhraseCollection::GetFilePos() const { return m_filePos; } diff --git a/OnDiskPt/TargetPhraseCollection.h b/OnDiskPt/TargetPhraseCollection.h index 72885ca12..227a0ffc2 100644 --- a/OnDiskPt/TargetPhraseCollection.h +++ b/OnDiskPt/TargetPhraseCollection.h @@ -46,7 +46,7 @@ class TargetPhraseCollection protected: typedef std::vector<TargetPhrase*> CollType; CollType m_coll; - UINT64 m_filePos; + uint64_t m_filePos; std::string m_debugStr; public: @@ -67,7 +67,7 @@ public: const TargetPhrase &GetTargetPhrase(size_t ind) const; - UINT64 GetFilePos() const; + uint64_t GetFilePos() const; Moses::TargetPhraseCollection *ConvertToMoses(const std::vector<Moses::FactorType> &inputFactors , const std::vector<Moses::FactorType> &outputFactors @@ -75,7 +75,7 @@ public: , const std::vector<float> &weightT , Vocab &vocab , bool isSyntax) const; - void ReadFromFile(size_t tableLimit, UINT64 filePos, OnDiskWrapper &onDiskWrapper); + void ReadFromFile(size_t tableLimit, uint64_t filePos, OnDiskWrapper &onDiskWrapper); const std::string GetDebugStr() const; void SetDebugStr(const std::string &str); diff --git a/OnDiskPt/Vocab.cpp b/OnDiskPt/Vocab.cpp index fbc63ac4e..dbe7b2e2f 100644 --- a/OnDiskPt/Vocab.cpp +++ b/OnDiskPt/Vocab.cpp @@ -38,7 +38,7 @@ bool Vocab::Load(OnDiskWrapper &onDiskWrapper) Moses::Tokenize(tokens, line); UTIL_THROW_IF2(tokens.size() != 2, "Vocab file corrupted"); const string &key = tokens[0]; - m_vocabColl[key] = Moses::Scan<UINT64>(tokens[1]); + m_vocabColl[key] = Moses::Scan<uint64_t>(tokens[1]); } // create lookup @@ -48,7 +48,7 @@ bool Vocab::Load(OnDiskWrapper &onDiskWrapper) CollType::const_iterator iter; for (iter = m_vocabColl.begin(); iter != m_vocabColl.end(); ++iter) { - UINT32 vocabId = iter->second; + uint32_t vocabId = iter->second; const std::string &word = iter->first; m_lookup[vocabId] = word; @@ -63,13 +63,13 @@ void Vocab::Save(OnDiskWrapper &onDiskWrapper) CollType::const_iterator iterVocab; for (iterVocab = m_vocabColl.begin(); iterVocab != m_vocabColl.end(); ++iterVocab) { const string &word = iterVocab->first; - UINT32 vocabId = iterVocab->second; + uint32_t vocabId = iterVocab->second; file << word << " " << vocabId << endl; } } -UINT64 Vocab::AddVocabId(const std::string &str) +uint64_t Vocab::AddVocabId(const std::string &str) { // find string id CollType::const_iterator iter = m_vocabColl.find(str); @@ -83,7 +83,7 @@ UINT64 Vocab::AddVocabId(const std::string &str) } } -UINT64 Vocab::GetVocabId(const std::string &str, bool &found) const +uint64_t Vocab::GetVocabId(const std::string &str, bool &found) const { // find string id CollType::const_iterator iter = m_vocabColl.find(str); diff --git a/OnDiskPt/Vocab.h b/OnDiskPt/Vocab.h index 56ec7c33b..2fa7ec635 100644 --- a/OnDiskPt/Vocab.h +++ b/OnDiskPt/Vocab.h @@ -34,19 +34,19 @@ class OnDiskWrapper; class Vocab { protected: - typedef std::map<std::string, UINT64> CollType; + typedef std::map<std::string, uint64_t> CollType; CollType m_vocabColl; std::vector<std::string> m_lookup; // opposite of m_vocabColl - UINT64 m_nextId; // starts @ 1 + uint64_t m_nextId; // starts @ 1 public: Vocab() :m_nextId(1) { } - UINT64 AddVocabId(const std::string &str); - UINT64 GetVocabId(const std::string &str, bool &found) const; - const std::string &GetString(UINT64 vocabId) const { + uint64_t AddVocabId(const std::string &str); + uint64_t GetVocabId(const std::string &str, bool &found) const; + const std::string &GetString(uint64_t vocabId) const { return m_lookup[vocabId]; } diff --git a/OnDiskPt/Word.cpp b/OnDiskPt/Word.cpp index 8932732ea..a68accb34 100644 --- a/OnDiskPt/Word.cpp +++ b/OnDiskPt/Word.cpp @@ -57,10 +57,10 @@ void Word::CreateFromString(const std::string &inString, Vocab &vocab) size_t Word::WriteToMemory(char *mem) const { - UINT64 *vocabMem = (UINT64*) mem; + uint64_t *vocabMem = (uint64_t*) mem; vocabMem[0] = m_vocabId; - size_t size = sizeof(UINT64); + size_t size = sizeof(uint64_t); // is non-term char bNonTerm = (char) m_isNonTerminal; @@ -72,10 +72,10 @@ size_t Word::WriteToMemory(char *mem) const size_t Word::ReadFromMemory(const char *mem) { - UINT64 *vocabMem = (UINT64*) mem; + uint64_t *vocabMem = (uint64_t*) mem; m_vocabId = vocabMem[0]; - size_t memUsed = sizeof(UINT64); + size_t memUsed = sizeof(uint64_t); // is non-term char bNonTerm; @@ -88,8 +88,8 @@ size_t Word::ReadFromMemory(const char *mem) size_t Word::ReadFromFile(std::fstream &file) { - const size_t memAlloc = sizeof(UINT64) + sizeof(char); - char mem[sizeof(UINT64) + sizeof(char)]; + const size_t memAlloc = sizeof(uint64_t) + sizeof(char); + char mem[sizeof(uint64_t) + sizeof(char)]; file.read(mem, memAlloc); size_t memUsed = ReadFromMemory(mem); diff --git a/OnDiskPt/Word.h b/OnDiskPt/Word.h index 53253fa24..39ebf336b 100644 --- a/OnDiskPt/Word.h +++ b/OnDiskPt/Word.h @@ -43,7 +43,7 @@ class Word private: bool m_isNonTerminal; - UINT64 m_vocabId; + uint64_t m_vocabId; public: explicit Word() { @@ -67,7 +67,7 @@ public: size_t ReadFromMemory(const char *mem); size_t ReadFromFile(std::fstream &file); - void SetVocabId(UINT32 vocabId) { + void SetVocabId(uint32_t vocabId) { m_vocabId = vocabId; } diff --git a/moses/File.h b/moses/File.h index cd15165e3..fbf666ef9 100644 --- a/moses/File.h +++ b/moses/File.h @@ -57,7 +57,7 @@ template<typename T> inline void fRead(FILE* f,T& t) template<typename T> inline size_t fWrite(FILE* f,const T* b,const T* e) { - UINT32 s=std::distance(b,e); + uint32_t s=std::distance(b,e); size_t rv=fWrite(f,s); if(fwrite(b,sizeof(T),s,f)!=s) { UTIL_THROW2("ERROR: fwrite!"); @@ -67,7 +67,7 @@ template<typename T> inline size_t fWrite(FILE* f,const T* b,const T* e) template<typename T> inline size_t fWrite(FILE* f,const T b,const T e) { - UINT32 s=std::distance(b,e); + uint32_t s=std::distance(b,e); size_t rv=fWrite(f,s); if(fwrite(&(*b),sizeof(T),s,f)!=s) { UTIL_THROW2("ERROR: fwrite!"); @@ -77,7 +77,7 @@ template<typename T> inline size_t fWrite(FILE* f,const T b,const T e) template<typename C> inline size_t fWriteVector(FILE* f,const C& v) { - UINT32 s=v.size(); + uint32_t s=v.size(); size_t rv=fWrite(f,s); if(fwrite(&v[0],sizeof(typename C::value_type),s,f)!=s) { UTIL_THROW2("ERROR: fwrite!"); @@ -87,7 +87,7 @@ template<typename C> inline size_t fWriteVector(FILE* f,const C& v) template<typename C> inline void fReadVector(FILE* f, C& v) { - UINT32 s; + uint32_t s; fRead(f,s); v.resize(s); size_t r=fread(&(*v.begin()),sizeof(typename C::value_type),s,f); @@ -96,7 +96,7 @@ template<typename C> inline void fReadVector(FILE* f, C& v) } } -inline size_t fWriteString(FILE* f,const char* e, UINT32 s) +inline size_t fWriteString(FILE* f,const char* e, uint32_t s) { size_t rv=fWrite(f,s); if(fwrite(e,sizeof(char),s,f)!=s) { @@ -107,7 +107,7 @@ inline size_t fWriteString(FILE* f,const char* e, UINT32 s) inline void fReadString(FILE* f,std::string& e) { - UINT32 s; + uint32_t s; fRead(f,s); char* a=new char[s+1]; if(fread(a,sizeof(char),s,f)!=s) { @@ -120,7 +120,7 @@ inline void fReadString(FILE* f,std::string& e) inline size_t fWriteStringVector(FILE* f,const std::vector<std::string>& v) { - UINT32 s=v.size(); + uint32_t s=v.size(); size_t totrv=fWrite(f,s); for (size_t i=0; i<s; i++) { totrv+=fWriteString(f,v.at(i).c_str(),v.at(i).size()); @@ -130,7 +130,7 @@ inline size_t fWriteStringVector(FILE* f,const std::vector<std::string>& v) inline void fReadStringVector(FILE* f, std::vector<std::string>& v) { - UINT32 s; + uint32_t s; fRead(f,s); v.resize(s); diff --git a/moses/PrefixTreeMap.cpp b/moses/PrefixTreeMap.cpp index 1719ebdba..fc51899f3 100644 --- a/moses/PrefixTreeMap.cpp +++ b/moses/PrefixTreeMap.cpp @@ -13,14 +13,14 @@ void GenericCandidate::readBin(FILE* f) { m_PhraseList.clear(); m_ScoreList.clear(); - UINT32 num_phrases; // on older compilers, <stdint.h> may need to be included + uint32_t num_phrases; // on older compilers, <stdint.h> may need to be included fRead(f, num_phrases); for(unsigned int i = 0; i < num_phrases; ++i) { IPhrase phrase; fReadVector(f, phrase); m_PhraseList.push_back(phrase); }; - UINT32 num_scores; + uint32_t num_scores; fRead(f, num_scores); for(unsigned int j = 0; j < num_scores; ++j) { std::vector<float> score; @@ -32,11 +32,11 @@ void GenericCandidate::readBin(FILE* f) void GenericCandidate::writeBin(FILE* f) const { // cast is necessary to ensure compatibility between 32- and 64-bit platforms - fWrite(f, static_cast<UINT32>(m_PhraseList.size())); + fWrite(f, static_cast<uint32_t>(m_PhraseList.size())); for(size_t i = 0; i < m_PhraseList.size(); ++i) { fWriteVector(f, m_PhraseList[i]); } - fWrite(f, static_cast<UINT32>(m_ScoreList.size())); + fWrite(f, static_cast<uint32_t>(m_ScoreList.size())); for(size_t j = 0; j < m_ScoreList.size(); ++j) { fWriteVector(f, m_ScoreList[j]); } @@ -45,7 +45,7 @@ void GenericCandidate::writeBin(FILE* f) const void Candidates::writeBin(FILE* f) const { - UINT32 s = this->size(); + uint32_t s = this->size(); fWrite(f,s); for(size_t i = 0; i < s; ++i) { MyBase::operator[](i).writeBin(f); @@ -54,7 +54,7 @@ void Candidates::writeBin(FILE* f) const void Candidates::readBin(FILE* f) { - UINT32 s; + uint32_t s; fRead(f,s); this->resize(s); for(size_t i = 0; i<s; ++i) { diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp index 77678e523..349fc4cbc 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp @@ -64,7 +64,7 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk( ChartRuleLookupManagerOnDisk::~ChartRuleLookupManagerOnDisk() { - std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache; + std::map<uint64_t, const TargetPhraseCollection*>::const_iterator iterCache; for (iterCache = m_cache.begin(); iterCache != m_cache.end(); ++iterCache) { delete iterCache->second; } @@ -239,8 +239,8 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection( const TargetPhraseCollection *targetPhraseCollection = NULL; const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper); if (node) { - UINT64 tpCollFilePos = node->GetValue(); - std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache = m_cache.find(tpCollFilePos); + uint64_t tpCollFilePos = node->GetValue(); + std::map<uint64_t, const TargetPhraseCollection*>::const_iterator iterCache = m_cache.find(tpCollFilePos); if (iterCache == m_cache.end()) { const OnDiskPt::TargetPhraseCollection *tpcollBerkeleyDb = node->GetTargetPhraseCollection(m_dictionary.GetTableLimit(), m_dbWrapper); diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h index 0ff87b378..42cbdcc46 100644 --- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h +++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h @@ -55,7 +55,7 @@ private: const std::vector<FactorType> &m_inputFactorsVec; const std::vector<FactorType> &m_outputFactorsVec; std::vector<DottedRuleStackOnDisk*> m_expandableDottedRuleListVec; - std::map<UINT64, const TargetPhraseCollection*> m_cache; + std::map<uint64_t, const TargetPhraseCollection*> m_cache; std::list<const OnDiskPt::PhraseNode*> m_sourcePhraseNode; }; diff --git a/moses/TypeDef.h b/moses/TypeDef.h index 59cb91499..a47a6e2fe 100644 --- a/moses/TypeDef.h +++ b/moses/TypeDef.h @@ -23,17 +23,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA #include <limits> #include <vector> #include <string> +#include <stdint.h> //! all the typedefs and enums goes here -#ifdef WIN32 -#include <BaseTsd.h> -#else -#include <stdint.h> - -typedef uint32_t UINT32; -typedef uint64_t UINT64; -#endif namespace Moses { |