diff options
author | Tetsuo Kiso <tetsuo-s@is.naist.jp> | 2012-12-06 21:39:02 +0400 |
---|---|---|
committer | Tetsuo Kiso <tetsuo-s@is.naist.jp> | 2012-12-06 21:39:02 +0400 |
commit | c7f6e3832615e92bc1321c2fb868064271c68d7e (patch) | |
tree | fd769ecbecbfa26c53c8800710326be971af9a98 /mert/Data.cpp | |
parent | 38e145e556274a199e270cc9ede833491b0f1a86 (diff) |
Use FilePiece to load N-best lists.
Since FilePiece is friendly with StringPiece.
Diffstat (limited to 'mert/Data.cpp')
-rw-r--r-- | mert/Data.cpp | 73 |
1 files changed, 38 insertions, 35 deletions
diff --git a/mert/Data.cpp b/mert/Data.cpp index 7ae506326..bbc8827cb 100644 --- a/mert/Data.cpp +++ b/mert/Data.cpp @@ -11,12 +11,12 @@ #include <fstream> #include "Data.h" -#include "FileStream.h" #include "Scorer.h" #include "ScorerFactory.h" #include "Util.h" #include "util/check.hh" +#include "util/file_piece.hh" #include "util/tokenize_piece.hh" #include "util/string_piece.hh" #include "FeatureDataIterator.h" @@ -136,57 +136,60 @@ void Data::load(const std::string &featfile, const std::string &scorefile) { void Data::loadNBest(const string &file) { TRACE_ERR("loading nbest from " << file << endl); - inputfilestream inp(file); // matches a stream with a file. Opens the file - if (!inp.good()) - throw runtime_error("Unable to open: " + file); + util::FilePiece in(file.c_str()); ScoreStats scoreentry; - string line, sentence, feature_str, alignment; + string sentence, feature_str, alignment; int sentence_index; - while (getline(inp, line, '\n')) { - if (line.empty()) continue; - // adding statistics for error measures - scoreentry.clear(); + while (true) { + try { + StringPiece line = in.ReadLine(); + if (line.empty()) continue; + // adding statistics for error measures + scoreentry.clear(); - util::TokenIter<util::MultiCharacter> it(line, util::MultiCharacter("|||")); + util::TokenIter<util::MultiCharacter> it(line, util::MultiCharacter("|||")); - sentence_index = ParseInt(*it); - ++it; - sentence = it->as_string(); - ++it; - feature_str = it->as_string(); - ++it; - - if (it) { - ++it; // skip model score. + sentence_index = ParseInt(*it); + ++it; + sentence = it->as_string(); + ++it; + feature_str = it->as_string(); + ++it; if (it) { - ++it; - alignment = it->as_string(); //fifth field (if present) is either phrase or word alignment + ++it; // skip model score. + if (it) { ++it; - alignment = it->as_string(); //sixth field (if present) is word alignment + alignment = it->as_string(); //fifth field (if present) is either phrase or word alignment + if (it) { + ++it; + alignment = it->as_string(); //sixth field (if present) is word alignment + } } } - } - //TODO check alignment exists if scorers need it + //TODO check alignment exists if scorers need it - if (m_scorer->useAlignment()) { - sentence += "|||"; - sentence += alignment; - } - m_scorer->prepareStats(sentence_index, sentence, scoreentry); + if (m_scorer->useAlignment()) { + sentence += "|||"; + sentence += alignment; + } + m_scorer->prepareStats(sentence_index, sentence, scoreentry); - m_score_data->add(scoreentry, sentence_index); + m_score_data->add(scoreentry, sentence_index); - // examine first line for name of features - if (!existsFeatureNames()) { - InitFeatureMap(feature_str); + // examine first line for name of features + if (!existsFeatureNames()) { + InitFeatureMap(feature_str); + } + AddFeatures(feature_str, sentence_index); + } catch (util::EndOfFileException &e) { + PrintUserTime("Loaded N-best lists"); + break; } - AddFeatures(feature_str, sentence_index); } - inp.close(); } void Data::save(const std::string &featfile, const std::string &scorefile, bool bin) { |