Welcome to mirror list, hosted at ThFree Co, Russian Federation.

github.com/moses-smt/mosesdecoder.git - Unnamed repository; edit this file 'description' to name the repository.
summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--OnDiskPt/OnDiskWrapper.cpp14
-rw-r--r--OnDiskPt/OnDiskWrapper.h4
-rw-r--r--OnDiskPt/PhraseNode.cpp42
-rw-r--r--OnDiskPt/PhraseNode.h18
-rw-r--r--OnDiskPt/TargetPhrase.cpp94
-rw-r--r--OnDiskPt/TargetPhrase.h16
-rw-r--r--OnDiskPt/TargetPhraseCollection.cpp28
-rw-r--r--OnDiskPt/TargetPhraseCollection.h6
-rw-r--r--OnDiskPt/Vocab.cpp10
-rw-r--r--OnDiskPt/Vocab.h10
-rw-r--r--OnDiskPt/Word.cpp12
-rw-r--r--OnDiskPt/Word.h4
-rw-r--r--moses/File.h16
-rw-r--r--moses/PrefixTreeMap.cpp12
-rw-r--r--moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp6
-rw-r--r--moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h2
-rw-r--r--moses/TypeDef.h9
17 files changed, 148 insertions, 155 deletions
diff --git a/OnDiskPt/OnDiskWrapper.cpp b/OnDiskPt/OnDiskWrapper.cpp
index 91935e965..3bfb7d2b1 100644
--- a/OnDiskPt/OnDiskWrapper.cpp
+++ b/OnDiskPt/OnDiskWrapper.cpp
@@ -51,7 +51,7 @@ void OnDiskWrapper::BeginLoad(const std::string &filePath)
if (!m_vocab.Load(*this))
UTIL_THROW(util::FileOpenException, "Couldn't load vocab");
- UINT64 rootFilePos = GetMisc("RootNodeOffset");
+ uint64_t rootFilePos = GetMisc("RootNodeOffset");
m_rootSourceNode = new PhraseNode(rootFilePos, *this);
}
@@ -102,7 +102,7 @@ bool OnDiskWrapper::LoadMisc()
const string &key = tokens[0];
- m_miscInfo[key] = Moses::Scan<UINT64>(tokens[1]);
+ m_miscInfo[key] = Moses::Scan<uint64_t>(tokens[1]);
}
return true;
@@ -199,17 +199,17 @@ void OnDiskWrapper::SaveMisc()
size_t OnDiskWrapper::GetSourceWordSize() const
{
- return sizeof(UINT64) + sizeof(char);
+ return sizeof(uint64_t) + sizeof(char);
}
size_t OnDiskWrapper::GetTargetWordSize() const
{
- return sizeof(UINT64) + sizeof(char);
+ return sizeof(uint64_t) + sizeof(char);
}
-UINT64 OnDiskWrapper::GetMisc(const std::string &key) const
+uint64_t OnDiskWrapper::GetMisc(const std::string &key) const
{
- std::map<std::string, UINT64>::const_iterator iter;
+ std::map<std::string, uint64_t>::const_iterator iter;
iter = m_miscInfo.find(key);
UTIL_THROW_IF2(iter == m_miscInfo.end()
, "Couldn't find value for key " << key
@@ -243,7 +243,7 @@ Word *OnDiskWrapper::ConvertFromMoses(const std::vector<Moses::FactorType> &fact
} // for (size_t factorType
bool found;
- UINT64 vocabId = m_vocab.GetVocabId(strme.str(), found);
+ uint64_t vocabId = m_vocab.GetVocabId(strme.str(), found);
if (!found) {
// factor not in phrase table -> phrse definately not in. exit
delete newWord;
diff --git a/OnDiskPt/OnDiskWrapper.h b/OnDiskPt/OnDiskWrapper.h
index 5b7cdfe01..bf398506c 100644
--- a/OnDiskPt/OnDiskWrapper.h
+++ b/OnDiskPt/OnDiskWrapper.h
@@ -43,7 +43,7 @@ protected:
size_t m_defaultNodeSize;
PhraseNode *m_rootSourceNode;
- std::map<std::string, UINT64> m_miscInfo;
+ std::map<std::string, uint64_t> m_miscInfo;
void SaveMisc();
bool OpenForLoad(const std::string &filePath);
@@ -105,7 +105,7 @@ public:
return *m_rootSourceNode;
}
- UINT64 GetMisc(const std::string &key) const;
+ uint64_t GetMisc(const std::string &key) const;
Word *ConvertFromMoses(const std::vector<Moses::FactorType> &factorsVec
, const Moses::Word &origWord) const;
diff --git a/OnDiskPt/PhraseNode.cpp b/OnDiskPt/PhraseNode.cpp
index 01cfda847..8e50147b2 100644
--- a/OnDiskPt/PhraseNode.cpp
+++ b/OnDiskPt/PhraseNode.cpp
@@ -31,8 +31,8 @@ namespace OnDiskPt
size_t PhraseNode::GetNodeSize(size_t numChildren, size_t wordSize, size_t countSize)
{
- size_t ret = sizeof(UINT64) * 2 // num children, value
- + (wordSize + sizeof(UINT64)) * numChildren // word + ptr to next source node
+ size_t ret = sizeof(uint64_t) * 2 // num children, value
+ + (wordSize + sizeof(uint64_t)) * numChildren // word + ptr to next source node
+ sizeof(float) * countSize; // count info
return ret;
}
@@ -45,7 +45,7 @@ PhraseNode::PhraseNode()
{
}
-PhraseNode::PhraseNode(UINT64 filePos, OnDiskWrapper &onDiskWrapper)
+PhraseNode::PhraseNode(uint64_t filePos, OnDiskWrapper &onDiskWrapper)
:m_counts(onDiskWrapper.GetNumCounts())
{
// load saved node
@@ -55,26 +55,26 @@ PhraseNode::PhraseNode(UINT64 filePos, OnDiskWrapper &onDiskWrapper)
std::fstream &file = onDiskWrapper.GetFileSource();
file.seekg(filePos);
- assert(filePos == (UINT64)file.tellg());
+ assert(filePos == (uint64_t)file.tellg());
- file.read((char*) &m_numChildrenLoad, sizeof(UINT64));
+ file.read((char*) &m_numChildrenLoad, sizeof(uint64_t));
size_t memAlloc = GetNodeSize(m_numChildrenLoad, onDiskWrapper.GetSourceWordSize(), countSize);
m_memLoad = (char*) malloc(memAlloc);
// go to start of node again
file.seekg(filePos);
- assert(filePos == (UINT64)file.tellg());
+ assert(filePos == (uint64_t)file.tellg());
// read everything into memory
file.read(m_memLoad, memAlloc);
- assert(filePos + memAlloc == (UINT64)file.tellg());
+ assert(filePos + memAlloc == (uint64_t)file.tellg());
// get value
- m_value = ((UINT64*)m_memLoad)[1];
+ m_value = ((uint64_t*)m_memLoad)[1];
// get counts
- float *memFloat = (float*) (m_memLoad + sizeof(UINT64) * 2);
+ float *memFloat = (float*) (m_memLoad + sizeof(uint64_t) * 2);
assert(countSize == 1);
m_counts[0] = memFloat[0];
@@ -108,10 +108,10 @@ void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimi
//memset(mem, 0xfe, memAlloc);
size_t memUsed = 0;
- UINT64 *memArray = (UINT64*) mem;
+ uint64_t *memArray = (uint64_t*) mem;
memArray[0] = GetSize(); // num of children
memArray[1] = m_value; // file pos of corresponding target phrases
- memUsed += 2 * sizeof(UINT64);
+ memUsed += 2 * sizeof(uint64_t);
// count info
float *memFloat = (float*) (mem + memUsed);
@@ -133,9 +133,9 @@ void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimi
size_t wordMemUsed = childWord.WriteToMemory(currMem);
memUsed += wordMemUsed;
- UINT64 *memArray = (UINT64*) (mem + memUsed);
+ uint64_t *memArray = (uint64_t*) (mem + memUsed);
memArray[0] = childNode.GetFilePos();
- memUsed += sizeof(UINT64);
+ memUsed += sizeof(uint64_t);
}
@@ -148,7 +148,7 @@ void PhraseNode::Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimi
file.seekp(0, ios::end);
file.write(mem, memUsed);
- UINT64 endPos = file.tellp();
+ uint64_t endPos = file.tellp();
assert(m_filePos + memUsed == endPos);
free(mem);
@@ -206,7 +206,7 @@ const PhraseNode *PhraseNode::GetChild(const Word &wordSought, OnDiskWrapper &on
x = (l + r) / 2;
Word wordFound;
- UINT64 childFilePos;
+ uint64_t childFilePos;
GetChild(wordFound, childFilePos, x, onDiskWrapper);
if (wordSought == wordFound) {
@@ -222,14 +222,14 @@ const PhraseNode *PhraseNode::GetChild(const Word &wordSought, OnDiskWrapper &on
return ret;
}
-void PhraseNode::GetChild(Word &wordFound, UINT64 &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const
+void PhraseNode::GetChild(Word &wordFound, uint64_t &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const
{
size_t wordSize = onDiskWrapper.GetSourceWordSize();
- size_t childSize = wordSize + sizeof(UINT64);
+ size_t childSize = wordSize + sizeof(uint64_t);
char *currMem = m_memLoad
- + sizeof(UINT64) * 2 // size & file pos of target phrase coll
+ + sizeof(uint64_t) * 2 // size & file pos of target phrase coll
+ sizeof(float) * onDiskWrapper.GetNumCounts() // count info
+ childSize * ind;
@@ -237,15 +237,15 @@ void PhraseNode::GetChild(Word &wordFound, UINT64 &childFilePos, size_t ind, OnD
assert(memRead == childSize);
}
-size_t PhraseNode::ReadChild(Word &wordFound, UINT64 &childFilePos, const char *mem) const
+size_t PhraseNode::ReadChild(Word &wordFound, uint64_t &childFilePos, const char *mem) const
{
size_t memRead = wordFound.ReadFromMemory(mem);
const char *currMem = mem + memRead;
- UINT64 *memArray = (UINT64*) (currMem);
+ uint64_t *memArray = (uint64_t*) (currMem);
childFilePos = memArray[0];
- memRead += sizeof(UINT64);
+ memRead += sizeof(uint64_t);
return memRead;
}
diff --git a/OnDiskPt/PhraseNode.h b/OnDiskPt/PhraseNode.h
index 6b629a401..901852952 100644
--- a/OnDiskPt/PhraseNode.h
+++ b/OnDiskPt/PhraseNode.h
@@ -36,7 +36,7 @@ class PhraseNode
{
friend std::ostream& operator<<(std::ostream&, const PhraseNode&);
protected:
- UINT64 m_filePos, m_value;
+ uint64_t m_filePos, m_value;
typedef std::map<Word, PhraseNode> ChildColl;
ChildColl m_children;
@@ -48,35 +48,35 @@ protected:
TargetPhraseCollection m_targetPhraseColl;
char *m_memLoad, *m_memLoadLast;
- UINT64 m_numChildrenLoad;
+ uint64_t m_numChildrenLoad;
void AddTargetPhrase(size_t pos, const SourcePhrase &sourcePhrase
, TargetPhrase *targetPhrase, OnDiskWrapper &onDiskWrapper
, size_t tableLimit, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort);
- size_t ReadChild(Word &wordFound, UINT64 &childFilePos, const char *mem) const;
- void GetChild(Word &wordFound, UINT64 &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const;
+ size_t ReadChild(Word &wordFound, uint64_t &childFilePos, const char *mem) const;
+ void GetChild(Word &wordFound, uint64_t &childFilePos, size_t ind, OnDiskWrapper &onDiskWrapper) const;
public:
static size_t GetNodeSize(size_t numChildren, size_t wordSize, size_t countSize);
PhraseNode(); // unsaved node
- PhraseNode(UINT64 filePos, OnDiskWrapper &onDiskWrapper); // load saved node
+ PhraseNode(uint64_t filePos, OnDiskWrapper &onDiskWrapper); // load saved node
~PhraseNode();
- void Add(const Word &word, UINT64 nextFilePos, size_t wordSize);
+ void Add(const Word &word, uint64_t nextFilePos, size_t wordSize);
void Save(OnDiskWrapper &onDiskWrapper, size_t pos, size_t tableLimit);
void AddTargetPhrase(const SourcePhrase &sourcePhrase, TargetPhrase *targetPhrase
, OnDiskWrapper &onDiskWrapper, size_t tableLimit
, const std::vector<float> &counts, OnDiskPt::PhrasePtr spShort);
- UINT64 GetFilePos() const {
+ uint64_t GetFilePos() const {
return m_filePos;
}
- UINT64 GetValue() const {
+ uint64_t GetValue() const {
return m_value;
}
- void SetValue(UINT64 value) {
+ void SetValue(uint64_t value) {
m_value = value;
}
size_t GetSize() const {
diff --git a/OnDiskPt/TargetPhrase.cpp b/OnDiskPt/TargetPhrase.cpp
index 5d1dc8e5b..f0501a937 100644
--- a/OnDiskPt/TargetPhrase.cpp
+++ b/OnDiskPt/TargetPhrase.cpp
@@ -103,17 +103,17 @@ char *TargetPhrase::WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed)
size_t spSize = sp->GetSize();
size_t sourceWordSize = onDiskWrapper.GetSourceWordSize();
- size_t memNeeded = sizeof(UINT64) // num of words
+ size_t memNeeded = sizeof(uint64_t) // num of words
+ targetWordSize * phraseSize // actual words. lhs as last words
- + sizeof(UINT64) // num source words
+ + sizeof(uint64_t) // num source words
+ sourceWordSize * spSize; // actual source words
memUsed = 0;
- UINT64 *mem = (UINT64*) malloc(memNeeded);
+ uint64_t *mem = (uint64_t*) malloc(memNeeded);
// write size
mem[0] = phraseSize;
- memUsed += sizeof(UINT64);
+ memUsed += sizeof(uint64_t);
// write each word
for (size_t pos = 0; pos < phraseSize; ++pos) {
@@ -124,9 +124,9 @@ char *TargetPhrase::WriteToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed)
// write size of source phrase and all source words
char *currPtr = (char*)mem + memUsed;
- UINT64 *memTmp = (UINT64*) currPtr;
+ uint64_t *memTmp = (uint64_t*) currPtr;
memTmp[0] = spSize;
- memUsed += sizeof(UINT64);
+ memUsed += sizeof(uint64_t);
for (size_t pos = 0; pos < spSize; ++pos) {
const Word &word = sp->GetWord(pos);
char *currPtr = (char*)mem + memUsed;
@@ -145,13 +145,13 @@ void TargetPhrase::Save(OnDiskWrapper &onDiskWrapper)
std::fstream &file = onDiskWrapper.GetFileTargetInd();
- UINT64 startPos = file.tellp();
+ uint64_t startPos = file.tellp();
file.seekp(0, ios::end);
file.write(mem, memUsed);
#ifndef NDEBUG
- UINT64 endPos = file.tellp();
+ uint64_t endPos = file.tellp();
assert(startPos + memUsed == endPos);
#endif
@@ -167,11 +167,11 @@ char *TargetPhrase::WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t
size_t sparseFeatureSize = m_sparseFeatures.size();
size_t propSize = m_property.size();
- size_t memNeeded = sizeof(UINT64) // file pos (phrase id)
- + sizeof(UINT64) + 2 * sizeof(UINT64) * numAlign // align
+ size_t memNeeded = sizeof(uint64_t) // file pos (phrase id)
+ + sizeof(uint64_t) + 2 * sizeof(uint64_t) * numAlign // align
+ sizeof(float) * numScores // scores
- + sizeof(UINT64) + sparseFeatureSize // sparse features string
- + sizeof(UINT64) + propSize; // property string
+ + sizeof(uint64_t) + sparseFeatureSize // sparse features string
+ + sizeof(uint64_t) + propSize; // property string
char *mem = (char*) malloc(memNeeded);
//memset(mem, 0, memNeeded);
@@ -179,8 +179,8 @@ char *TargetPhrase::WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t
memUsed = 0;
// phrase id
- memcpy(mem, &m_filePos, sizeof(UINT64));
- memUsed += sizeof(UINT64);
+ memcpy(mem, &m_filePos, sizeof(uint64_t));
+ memUsed += sizeof(uint64_t);
// align
size_t tmp = WriteAlignToMemory(mem + memUsed);
@@ -203,11 +203,11 @@ char *TargetPhrase::WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t
size_t TargetPhrase::WriteStringToMemory(char *mem, const std::string &str) const
{
size_t memUsed = 0;
- UINT64 *memTmp = (UINT64*) mem;
+ uint64_t *memTmp = (uint64_t*) mem;
size_t strSize = str.size();
memTmp[0] = strSize;
- memUsed += sizeof(UINT64);
+ memUsed += sizeof(uint64_t);
const char *charStr = str.c_str();
memcpy(mem + memUsed, charStr, strSize);
@@ -221,7 +221,7 @@ size_t TargetPhrase::WriteAlignToMemory(char *mem) const
size_t memUsed = 0;
// num of alignments
- UINT64 numAlign = m_align.size();
+ uint64_t numAlign = m_align.size();
memcpy(mem, &numAlign, sizeof(numAlign));
memUsed += sizeof(numAlign);
@@ -319,20 +319,20 @@ Moses::TargetPhrase *TargetPhrase::ConvertToMoses(const std::vector<Moses::Facto
return ret;
}
-UINT64 TargetPhrase::ReadOtherInfoFromFile(UINT64 filePos, std::fstream &fileTPColl)
+uint64_t TargetPhrase::ReadOtherInfoFromFile(uint64_t filePos, std::fstream &fileTPColl)
{
- assert(filePos == (UINT64)fileTPColl.tellg());
+ assert(filePos == (uint64_t)fileTPColl.tellg());
- UINT64 memUsed = 0;
- fileTPColl.read((char*) &m_filePos, sizeof(UINT64));
- memUsed += sizeof(UINT64);
+ uint64_t memUsed = 0;
+ fileTPColl.read((char*) &m_filePos, sizeof(uint64_t));
+ memUsed += sizeof(uint64_t);
assert(m_filePos != 0);
memUsed += ReadAlignFromFile(fileTPColl);
- assert((memUsed + filePos) == (UINT64)fileTPColl.tellg());
+ assert((memUsed + filePos) == (uint64_t)fileTPColl.tellg());
memUsed += ReadScoresFromFile(fileTPColl);
- assert((memUsed + filePos) == (UINT64)fileTPColl.tellg());
+ assert((memUsed + filePos) == (uint64_t)fileTPColl.tellg());
// sparse features
memUsed += ReadStringFromFile(fileTPColl, m_sparseFeatures);
@@ -343,13 +343,13 @@ UINT64 TargetPhrase::ReadOtherInfoFromFile(UINT64 filePos, std::fstream &fileTPC
return memUsed;
}
-UINT64 TargetPhrase::ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr)
+uint64_t TargetPhrase::ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr)
{
- UINT64 bytesRead = 0;
+ uint64_t bytesRead = 0;
- UINT64 strSize;
- fileTPColl.read((char*) &strSize, sizeof(UINT64));
- bytesRead += sizeof(UINT64);
+ uint64_t strSize;
+ fileTPColl.read((char*) &strSize, sizeof(uint64_t));
+ bytesRead += sizeof(uint64_t);
if (strSize) {
char *mem = (char*) malloc(strSize + 1);
@@ -364,15 +364,15 @@ UINT64 TargetPhrase::ReadStringFromFile(std::fstream &fileTPColl, std::string &o
return bytesRead;
}
-UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP)
+uint64_t TargetPhrase::ReadFromFile(std::fstream &fileTP)
{
- UINT64 bytesRead = 0;
+ uint64_t bytesRead = 0;
fileTP.seekg(m_filePos);
- UINT64 numWords;
- fileTP.read((char*) &numWords, sizeof(UINT64));
- bytesRead += sizeof(UINT64);
+ uint64_t numWords;
+ fileTP.read((char*) &numWords, sizeof(uint64_t));
+ bytesRead += sizeof(uint64_t);
for (size_t ind = 0; ind < numWords; ++ind) {
WordPtr word(new Word());
@@ -381,9 +381,9 @@ UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP)
}
// read source words
- UINT64 numSourceWords;
- fileTP.read((char*) &numSourceWords, sizeof(UINT64));
- bytesRead += sizeof(UINT64);
+ uint64_t numSourceWords;
+ fileTP.read((char*) &numSourceWords, sizeof(uint64_t));
+ bytesRead += sizeof(uint64_t);
PhrasePtr sp(new SourcePhrase());
for (size_t ind = 0; ind < numSourceWords; ++ind) {
@@ -396,31 +396,31 @@ UINT64 TargetPhrase::ReadFromFile(std::fstream &fileTP)
return bytesRead;
}
-UINT64 TargetPhrase::ReadAlignFromFile(std::fstream &fileTPColl)
+uint64_t TargetPhrase::ReadAlignFromFile(std::fstream &fileTPColl)
{
- UINT64 bytesRead = 0;
+ uint64_t bytesRead = 0;
- UINT64 numAlign;
- fileTPColl.read((char*) &numAlign, sizeof(UINT64));
- bytesRead += sizeof(UINT64);
+ uint64_t numAlign;
+ fileTPColl.read((char*) &numAlign, sizeof(uint64_t));
+ bytesRead += sizeof(uint64_t);
for (size_t ind = 0; ind < numAlign; ++ind) {
AlignPair alignPair;
- fileTPColl.read((char*) &alignPair.first, sizeof(UINT64));
- fileTPColl.read((char*) &alignPair.second, sizeof(UINT64));
+ fileTPColl.read((char*) &alignPair.first, sizeof(uint64_t));
+ fileTPColl.read((char*) &alignPair.second, sizeof(uint64_t));
m_align.push_back(alignPair);
- bytesRead += sizeof(UINT64) * 2;
+ bytesRead += sizeof(uint64_t) * 2;
}
return bytesRead;
}
-UINT64 TargetPhrase::ReadScoresFromFile(std::fstream &fileTPColl)
+uint64_t TargetPhrase::ReadScoresFromFile(std::fstream &fileTPColl)
{
UTIL_THROW_IF2(m_scores.size() == 0, "Translation rules must must have some scores");
- UINT64 bytesRead = 0;
+ uint64_t bytesRead = 0;
for (size_t ind = 0; ind < m_scores.size(); ++ind) {
fileTPColl.read((char*) &m_scores[ind], sizeof(float));
diff --git a/OnDiskPt/TargetPhrase.h b/OnDiskPt/TargetPhrase.h
index efd91a4ae..b41b788d5 100644
--- a/OnDiskPt/TargetPhrase.h
+++ b/OnDiskPt/TargetPhrase.h
@@ -36,7 +36,7 @@ class Phrase;
namespace OnDiskPt
{
-typedef std::pair<UINT64, UINT64> AlignPair;
+typedef std::pair<uint64_t, uint64_t> AlignPair;
typedef std::vector<AlignPair> AlignType;
class Vocab;
@@ -53,15 +53,15 @@ protected:
std::string m_sparseFeatures, m_property;
std::vector<float> m_scores;
- UINT64 m_filePos;
+ uint64_t m_filePos;
size_t WriteAlignToMemory(char *mem) const;
size_t WriteScoresToMemory(char *mem) const;
size_t WriteStringToMemory(char *mem, const std::string &str) const;
- UINT64 ReadAlignFromFile(std::fstream &fileTPColl);
- UINT64 ReadScoresFromFile(std::fstream &fileTPColl);
- UINT64 ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr);
+ uint64_t ReadAlignFromFile(std::fstream &fileTPColl);
+ uint64_t ReadScoresFromFile(std::fstream &fileTPColl);
+ uint64_t ReadStringFromFile(std::fstream &fileTPColl, std::string &outStr);
public:
TargetPhrase() {
@@ -95,7 +95,7 @@ public:
char *WriteOtherInfoToMemory(OnDiskWrapper &onDiskWrapper, size_t &memUsed) const;
void Save(OnDiskWrapper &onDiskWrapper);
- UINT64 GetFilePos() const {
+ uint64_t GetFilePos() const {
return m_filePos;
}
float GetScore(size_t ind) const {
@@ -108,8 +108,8 @@ public:
, const Moses::PhraseDictionary &phraseDict
, const std::vector<float> &weightT
, bool isSyntax) const;
- UINT64 ReadOtherInfoFromFile(UINT64 filePos, std::fstream &fileTPColl);
- UINT64 ReadFromFile(std::fstream &fileTP);
+ uint64_t ReadOtherInfoFromFile(uint64_t filePos, std::fstream &fileTPColl);
+ uint64_t ReadFromFile(std::fstream &fileTP);
virtual void DebugPrint(std::ostream &out, const Vocab &vocab) const;
diff --git a/OnDiskPt/TargetPhraseCollection.cpp b/OnDiskPt/TargetPhraseCollection.cpp
index 7f6b5016f..73ad2540c 100644
--- a/OnDiskPt/TargetPhraseCollection.cpp
+++ b/OnDiskPt/TargetPhraseCollection.cpp
@@ -71,12 +71,12 @@ void TargetPhraseCollection::Save(OnDiskWrapper &onDiskWrapper)
{
std::fstream &file = onDiskWrapper.GetFileTargetColl();
- size_t memUsed = sizeof(UINT64);
+ size_t memUsed = sizeof(uint64_t);
char *mem = (char*) malloc(memUsed);
// size of coll
- UINT64 numPhrases = GetSize();
- ((UINT64*)mem)[0] = numPhrases;
+ uint64_t numPhrases = GetSize();
+ ((uint64_t*)mem)[0] = numPhrases;
// MAIN LOOP
CollType::iterator iter;
@@ -98,16 +98,16 @@ void TargetPhraseCollection::Save(OnDiskWrapper &onDiskWrapper)
}
// total number of bytes
- //((UINT64*)mem)[0] = (UINT64) memUsed;
+ //((uint64_t*)mem)[0] = (uint64_t) memUsed;
- UINT64 startPos = file.tellp();
+ uint64_t startPos = file.tellp();
file.seekp(0, ios::end);
file.write((char*) mem, memUsed);
free(mem);
#ifndef NDEBUG
- UINT64 endPos = file.tellp();
+ uint64_t endPos = file.tellp();
assert(startPos + memUsed == endPos);
#endif
m_filePos = startPos;
@@ -148,7 +148,7 @@ Moses::TargetPhraseCollection *TargetPhraseCollection::ConvertToMoses(const std:
}
-void TargetPhraseCollection::ReadFromFile(size_t tableLimit, UINT64 filePos, OnDiskWrapper &onDiskWrapper)
+void TargetPhraseCollection::ReadFromFile(size_t tableLimit, uint64_t filePos, OnDiskWrapper &onDiskWrapper)
{
fstream &fileTPColl = onDiskWrapper.GetFileTargetColl();
fstream &fileTP = onDiskWrapper.GetFileTargetInd();
@@ -156,23 +156,23 @@ void TargetPhraseCollection::ReadFromFile(size_t tableLimit, UINT64 filePos, OnD
size_t numScores = onDiskWrapper.GetNumScores();
- UINT64 numPhrases;
+ uint64_t numPhrases;
- UINT64 currFilePos = filePos;
+ uint64_t currFilePos = filePos;
fileTPColl.seekg(filePos);
- fileTPColl.read((char*) &numPhrases, sizeof(UINT64));
+ fileTPColl.read((char*) &numPhrases, sizeof(uint64_t));
// table limit
if (tableLimit) {
- numPhrases = std::min(numPhrases, (UINT64) tableLimit);
+ numPhrases = std::min(numPhrases, (uint64_t) tableLimit);
}
- currFilePos += sizeof(UINT64);
+ currFilePos += sizeof(uint64_t);
for (size_t ind = 0; ind < numPhrases; ++ind) {
TargetPhrase *tp = new TargetPhrase(numScores);
- UINT64 sizeOtherInfo = tp->ReadOtherInfoFromFile(currFilePos, fileTPColl);
+ uint64_t sizeOtherInfo = tp->ReadOtherInfoFromFile(currFilePos, fileTPColl);
tp->ReadFromFile(fileTP);
currFilePos += sizeOtherInfo;
@@ -181,7 +181,7 @@ void TargetPhraseCollection::ReadFromFile(size_t tableLimit, UINT64 filePos, OnD
}
}
-UINT64 TargetPhraseCollection::GetFilePos() const
+uint64_t TargetPhraseCollection::GetFilePos() const
{
return m_filePos;
}
diff --git a/OnDiskPt/TargetPhraseCollection.h b/OnDiskPt/TargetPhraseCollection.h
index 72885ca12..227a0ffc2 100644
--- a/OnDiskPt/TargetPhraseCollection.h
+++ b/OnDiskPt/TargetPhraseCollection.h
@@ -46,7 +46,7 @@ class TargetPhraseCollection
protected:
typedef std::vector<TargetPhrase*> CollType;
CollType m_coll;
- UINT64 m_filePos;
+ uint64_t m_filePos;
std::string m_debugStr;
public:
@@ -67,7 +67,7 @@ public:
const TargetPhrase &GetTargetPhrase(size_t ind) const;
- UINT64 GetFilePos() const;
+ uint64_t GetFilePos() const;
Moses::TargetPhraseCollection *ConvertToMoses(const std::vector<Moses::FactorType> &inputFactors
, const std::vector<Moses::FactorType> &outputFactors
@@ -75,7 +75,7 @@ public:
, const std::vector<float> &weightT
, Vocab &vocab
, bool isSyntax) const;
- void ReadFromFile(size_t tableLimit, UINT64 filePos, OnDiskWrapper &onDiskWrapper);
+ void ReadFromFile(size_t tableLimit, uint64_t filePos, OnDiskWrapper &onDiskWrapper);
const std::string GetDebugStr() const;
void SetDebugStr(const std::string &str);
diff --git a/OnDiskPt/Vocab.cpp b/OnDiskPt/Vocab.cpp
index fbc63ac4e..dbe7b2e2f 100644
--- a/OnDiskPt/Vocab.cpp
+++ b/OnDiskPt/Vocab.cpp
@@ -38,7 +38,7 @@ bool Vocab::Load(OnDiskWrapper &onDiskWrapper)
Moses::Tokenize(tokens, line);
UTIL_THROW_IF2(tokens.size() != 2, "Vocab file corrupted");
const string &key = tokens[0];
- m_vocabColl[key] = Moses::Scan<UINT64>(tokens[1]);
+ m_vocabColl[key] = Moses::Scan<uint64_t>(tokens[1]);
}
// create lookup
@@ -48,7 +48,7 @@ bool Vocab::Load(OnDiskWrapper &onDiskWrapper)
CollType::const_iterator iter;
for (iter = m_vocabColl.begin(); iter != m_vocabColl.end(); ++iter) {
- UINT32 vocabId = iter->second;
+ uint32_t vocabId = iter->second;
const std::string &word = iter->first;
m_lookup[vocabId] = word;
@@ -63,13 +63,13 @@ void Vocab::Save(OnDiskWrapper &onDiskWrapper)
CollType::const_iterator iterVocab;
for (iterVocab = m_vocabColl.begin(); iterVocab != m_vocabColl.end(); ++iterVocab) {
const string &word = iterVocab->first;
- UINT32 vocabId = iterVocab->second;
+ uint32_t vocabId = iterVocab->second;
file << word << " " << vocabId << endl;
}
}
-UINT64 Vocab::AddVocabId(const std::string &str)
+uint64_t Vocab::AddVocabId(const std::string &str)
{
// find string id
CollType::const_iterator iter = m_vocabColl.find(str);
@@ -83,7 +83,7 @@ UINT64 Vocab::AddVocabId(const std::string &str)
}
}
-UINT64 Vocab::GetVocabId(const std::string &str, bool &found) const
+uint64_t Vocab::GetVocabId(const std::string &str, bool &found) const
{
// find string id
CollType::const_iterator iter = m_vocabColl.find(str);
diff --git a/OnDiskPt/Vocab.h b/OnDiskPt/Vocab.h
index 56ec7c33b..2fa7ec635 100644
--- a/OnDiskPt/Vocab.h
+++ b/OnDiskPt/Vocab.h
@@ -34,19 +34,19 @@ class OnDiskWrapper;
class Vocab
{
protected:
- typedef std::map<std::string, UINT64> CollType;
+ typedef std::map<std::string, uint64_t> CollType;
CollType m_vocabColl;
std::vector<std::string> m_lookup; // opposite of m_vocabColl
- UINT64 m_nextId; // starts @ 1
+ uint64_t m_nextId; // starts @ 1
public:
Vocab()
:m_nextId(1) {
}
- UINT64 AddVocabId(const std::string &str);
- UINT64 GetVocabId(const std::string &str, bool &found) const;
- const std::string &GetString(UINT64 vocabId) const {
+ uint64_t AddVocabId(const std::string &str);
+ uint64_t GetVocabId(const std::string &str, bool &found) const;
+ const std::string &GetString(uint64_t vocabId) const {
return m_lookup[vocabId];
}
diff --git a/OnDiskPt/Word.cpp b/OnDiskPt/Word.cpp
index 8932732ea..a68accb34 100644
--- a/OnDiskPt/Word.cpp
+++ b/OnDiskPt/Word.cpp
@@ -57,10 +57,10 @@ void Word::CreateFromString(const std::string &inString, Vocab &vocab)
size_t Word::WriteToMemory(char *mem) const
{
- UINT64 *vocabMem = (UINT64*) mem;
+ uint64_t *vocabMem = (uint64_t*) mem;
vocabMem[0] = m_vocabId;
- size_t size = sizeof(UINT64);
+ size_t size = sizeof(uint64_t);
// is non-term
char bNonTerm = (char) m_isNonTerminal;
@@ -72,10 +72,10 @@ size_t Word::WriteToMemory(char *mem) const
size_t Word::ReadFromMemory(const char *mem)
{
- UINT64 *vocabMem = (UINT64*) mem;
+ uint64_t *vocabMem = (uint64_t*) mem;
m_vocabId = vocabMem[0];
- size_t memUsed = sizeof(UINT64);
+ size_t memUsed = sizeof(uint64_t);
// is non-term
char bNonTerm;
@@ -88,8 +88,8 @@ size_t Word::ReadFromMemory(const char *mem)
size_t Word::ReadFromFile(std::fstream &file)
{
- const size_t memAlloc = sizeof(UINT64) + sizeof(char);
- char mem[sizeof(UINT64) + sizeof(char)];
+ const size_t memAlloc = sizeof(uint64_t) + sizeof(char);
+ char mem[sizeof(uint64_t) + sizeof(char)];
file.read(mem, memAlloc);
size_t memUsed = ReadFromMemory(mem);
diff --git a/OnDiskPt/Word.h b/OnDiskPt/Word.h
index 53253fa24..39ebf336b 100644
--- a/OnDiskPt/Word.h
+++ b/OnDiskPt/Word.h
@@ -43,7 +43,7 @@ class Word
private:
bool m_isNonTerminal;
- UINT64 m_vocabId;
+ uint64_t m_vocabId;
public:
explicit Word() {
@@ -67,7 +67,7 @@ public:
size_t ReadFromMemory(const char *mem);
size_t ReadFromFile(std::fstream &file);
- void SetVocabId(UINT32 vocabId) {
+ void SetVocabId(uint32_t vocabId) {
m_vocabId = vocabId;
}
diff --git a/moses/File.h b/moses/File.h
index cd15165e3..fbf666ef9 100644
--- a/moses/File.h
+++ b/moses/File.h
@@ -57,7 +57,7 @@ template<typename T> inline void fRead(FILE* f,T& t)
template<typename T> inline size_t fWrite(FILE* f,const T* b,const T* e)
{
- UINT32 s=std::distance(b,e);
+ uint32_t s=std::distance(b,e);
size_t rv=fWrite(f,s);
if(fwrite(b,sizeof(T),s,f)!=s) {
UTIL_THROW2("ERROR: fwrite!");
@@ -67,7 +67,7 @@ template<typename T> inline size_t fWrite(FILE* f,const T* b,const T* e)
template<typename T> inline size_t fWrite(FILE* f,const T b,const T e)
{
- UINT32 s=std::distance(b,e);
+ uint32_t s=std::distance(b,e);
size_t rv=fWrite(f,s);
if(fwrite(&(*b),sizeof(T),s,f)!=s) {
UTIL_THROW2("ERROR: fwrite!");
@@ -77,7 +77,7 @@ template<typename T> inline size_t fWrite(FILE* f,const T b,const T e)
template<typename C> inline size_t fWriteVector(FILE* f,const C& v)
{
- UINT32 s=v.size();
+ uint32_t s=v.size();
size_t rv=fWrite(f,s);
if(fwrite(&v[0],sizeof(typename C::value_type),s,f)!=s) {
UTIL_THROW2("ERROR: fwrite!");
@@ -87,7 +87,7 @@ template<typename C> inline size_t fWriteVector(FILE* f,const C& v)
template<typename C> inline void fReadVector(FILE* f, C& v)
{
- UINT32 s;
+ uint32_t s;
fRead(f,s);
v.resize(s);
size_t r=fread(&(*v.begin()),sizeof(typename C::value_type),s,f);
@@ -96,7 +96,7 @@ template<typename C> inline void fReadVector(FILE* f, C& v)
}
}
-inline size_t fWriteString(FILE* f,const char* e, UINT32 s)
+inline size_t fWriteString(FILE* f,const char* e, uint32_t s)
{
size_t rv=fWrite(f,s);
if(fwrite(e,sizeof(char),s,f)!=s) {
@@ -107,7 +107,7 @@ inline size_t fWriteString(FILE* f,const char* e, UINT32 s)
inline void fReadString(FILE* f,std::string& e)
{
- UINT32 s;
+ uint32_t s;
fRead(f,s);
char* a=new char[s+1];
if(fread(a,sizeof(char),s,f)!=s) {
@@ -120,7 +120,7 @@ inline void fReadString(FILE* f,std::string& e)
inline size_t fWriteStringVector(FILE* f,const std::vector<std::string>& v)
{
- UINT32 s=v.size();
+ uint32_t s=v.size();
size_t totrv=fWrite(f,s);
for (size_t i=0; i<s; i++) {
totrv+=fWriteString(f,v.at(i).c_str(),v.at(i).size());
@@ -130,7 +130,7 @@ inline size_t fWriteStringVector(FILE* f,const std::vector<std::string>& v)
inline void fReadStringVector(FILE* f, std::vector<std::string>& v)
{
- UINT32 s;
+ uint32_t s;
fRead(f,s);
v.resize(s);
diff --git a/moses/PrefixTreeMap.cpp b/moses/PrefixTreeMap.cpp
index 1719ebdba..fc51899f3 100644
--- a/moses/PrefixTreeMap.cpp
+++ b/moses/PrefixTreeMap.cpp
@@ -13,14 +13,14 @@ void GenericCandidate::readBin(FILE* f)
{
m_PhraseList.clear();
m_ScoreList.clear();
- UINT32 num_phrases; // on older compilers, <stdint.h> may need to be included
+ uint32_t num_phrases; // on older compilers, <stdint.h> may need to be included
fRead(f, num_phrases);
for(unsigned int i = 0; i < num_phrases; ++i) {
IPhrase phrase;
fReadVector(f, phrase);
m_PhraseList.push_back(phrase);
};
- UINT32 num_scores;
+ uint32_t num_scores;
fRead(f, num_scores);
for(unsigned int j = 0; j < num_scores; ++j) {
std::vector<float> score;
@@ -32,11 +32,11 @@ void GenericCandidate::readBin(FILE* f)
void GenericCandidate::writeBin(FILE* f) const
{
// cast is necessary to ensure compatibility between 32- and 64-bit platforms
- fWrite(f, static_cast<UINT32>(m_PhraseList.size()));
+ fWrite(f, static_cast<uint32_t>(m_PhraseList.size()));
for(size_t i = 0; i < m_PhraseList.size(); ++i) {
fWriteVector(f, m_PhraseList[i]);
}
- fWrite(f, static_cast<UINT32>(m_ScoreList.size()));
+ fWrite(f, static_cast<uint32_t>(m_ScoreList.size()));
for(size_t j = 0; j < m_ScoreList.size(); ++j) {
fWriteVector(f, m_ScoreList[j]);
}
@@ -45,7 +45,7 @@ void GenericCandidate::writeBin(FILE* f) const
void Candidates::writeBin(FILE* f) const
{
- UINT32 s = this->size();
+ uint32_t s = this->size();
fWrite(f,s);
for(size_t i = 0; i < s; ++i) {
MyBase::operator[](i).writeBin(f);
@@ -54,7 +54,7 @@ void Candidates::writeBin(FILE* f) const
void Candidates::readBin(FILE* f)
{
- UINT32 s;
+ uint32_t s;
fRead(f,s);
this->resize(s);
for(size_t i = 0; i<s; ++i) {
diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp
index 77678e523..349fc4cbc 100644
--- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp
+++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.cpp
@@ -64,7 +64,7 @@ ChartRuleLookupManagerOnDisk::ChartRuleLookupManagerOnDisk(
ChartRuleLookupManagerOnDisk::~ChartRuleLookupManagerOnDisk()
{
- std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache;
+ std::map<uint64_t, const TargetPhraseCollection*>::const_iterator iterCache;
for (iterCache = m_cache.begin(); iterCache != m_cache.end(); ++iterCache) {
delete iterCache->second;
}
@@ -239,8 +239,8 @@ void ChartRuleLookupManagerOnDisk::GetChartRuleCollection(
const TargetPhraseCollection *targetPhraseCollection = NULL;
const OnDiskPt::PhraseNode *node = prevNode.GetChild(*sourceLHSBerkeleyDb, m_dbWrapper);
if (node) {
- UINT64 tpCollFilePos = node->GetValue();
- std::map<UINT64, const TargetPhraseCollection*>::const_iterator iterCache = m_cache.find(tpCollFilePos);
+ uint64_t tpCollFilePos = node->GetValue();
+ std::map<uint64_t, const TargetPhraseCollection*>::const_iterator iterCache = m_cache.find(tpCollFilePos);
if (iterCache == m_cache.end()) {
const OnDiskPt::TargetPhraseCollection *tpcollBerkeleyDb = node->GetTargetPhraseCollection(m_dictionary.GetTableLimit(), m_dbWrapper);
diff --git a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h
index 0ff87b378..42cbdcc46 100644
--- a/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h
+++ b/moses/TranslationModel/CYKPlusParser/ChartRuleLookupManagerOnDisk.h
@@ -55,7 +55,7 @@ private:
const std::vector<FactorType> &m_inputFactorsVec;
const std::vector<FactorType> &m_outputFactorsVec;
std::vector<DottedRuleStackOnDisk*> m_expandableDottedRuleListVec;
- std::map<UINT64, const TargetPhraseCollection*> m_cache;
+ std::map<uint64_t, const TargetPhraseCollection*> m_cache;
std::list<const OnDiskPt::PhraseNode*> m_sourcePhraseNode;
};
diff --git a/moses/TypeDef.h b/moses/TypeDef.h
index 59cb91499..a47a6e2fe 100644
--- a/moses/TypeDef.h
+++ b/moses/TypeDef.h
@@ -23,17 +23,10 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
#include <limits>
#include <vector>
#include <string>
+#include <stdint.h>
//! all the typedefs and enums goes here
-#ifdef WIN32
-#include <BaseTsd.h>
-#else
-#include <stdint.h>
-
-typedef uint32_t UINT32;
-typedef uint64_t UINT64;
-#endif
namespace Moses
{
generated by cgit v1.2.3 (git 2.25.1) at 2024-08-30 20:30:55 +0300