diff options
Diffstat (limited to 'contrib/moses2/TranslationModel/ProbingPT/vocabid.cpp')
-rw-r--r-- | contrib/moses2/TranslationModel/ProbingPT/vocabid.cpp | 59 |
1 files changed, 0 insertions, 59 deletions
diff --git a/contrib/moses2/TranslationModel/ProbingPT/vocabid.cpp b/contrib/moses2/TranslationModel/ProbingPT/vocabid.cpp deleted file mode 100644 index 696373ee5..000000000 --- a/contrib/moses2/TranslationModel/ProbingPT/vocabid.cpp +++ /dev/null @@ -1,59 +0,0 @@ -#include <boost/foreach.hpp> -#include "vocabid.hh" -#include "StoreVocab.h" -#include "../../legacy/Util2.h" - -namespace Moses2 -{ - -void add_to_map(StoreVocab<uint64_t> &sourceVocab, - const StringPiece &textin) -{ - //Tokenize - util::TokenIter<util::SingleCharacter> itWord(textin, util::SingleCharacter(' ')); - - while (itWord) { - StringPiece word = *itWord; - - util::TokenIter<util::SingleCharacter> itFactor(word, util::SingleCharacter('|')); - while (itFactor) { - StringPiece factor = *itFactor; - - sourceVocab.Insert(getHash(factor), factor.as_string()); - itFactor++; - } - itWord++; - } -} - -void serialize_map(const std::map<uint64_t, std::string> &karta, - const std::string &filename) -{ - std::ofstream os(filename.c_str()); - - std::map<uint64_t, std::string>::const_iterator iter; - for (iter = karta.begin(); iter != karta.end(); ++iter) { - os << iter->first << '\t' << iter->second << std::endl; - } - - os.close(); -} - -void read_map(std::map<uint64_t, std::string> &karta, const char* filename) -{ - std::ifstream is(filename); - - std::string line; - while (getline(is, line)) { - std::vector<std::string> toks = Tokenize(line, "\t"); - assert(toks.size() == 2); - uint64_t ind = Scan<uint64_t>(toks[1]); - karta[ind] = toks[0]; - } - - //Close the stream after we are done. - is.close(); -} - -} - |