diff options
author | Matous Machacek <machacekmatous@gmail.com> | 2012-01-24 22:18:44 +0400 |
---|---|---|
committer | Matous Machacek <machacekmatous@gmail.com> | 2012-01-24 22:18:44 +0400 |
commit | 6cbdfc513bfa0b3c49987b4929abfcd6400de766 (patch) | |
tree | d3927e3b1b885219a5374f1de74d0e7603771bd0 /mert/evaluator.cpp | |
parent | b4a50ec50bea5d81effd42058b07013eafa6b963 (diff) |
fixed bugs in mert/evaluator, nicer printing of results
Diffstat (limited to 'mert/evaluator.cpp')
-rw-r--r-- | mert/evaluator.cpp | 54 |
1 files changed, 41 insertions, 13 deletions
diff --git a/mert/evaluator.cpp b/mert/evaluator.cpp index c5a59c427..835fed017 100644 --- a/mert/evaluator.cpp +++ b/mert/evaluator.cpp @@ -22,19 +22,29 @@ float average(const vector<float>& list); float stdDeviation(const vector<float>& list, float avg); string int2string(int n); - +bool moreFiles = false; +bool moreScorers = false; void usage() { - cerr<<"usage: evaluator [options] --reference ref1[,ref2[,ref3...]] --candidate candFile "<<endl; - cerr<<"[--sctype|-s] the scorer type (default BLEU). You can specify more scorers (separated by ;)"<<endl; + cerr<<"usage: evaluator [options] --reference ref1[,ref2[,ref3...]] --candidate cand1[,cand2[,cand3...]] "<<endl; + cerr<<"[--sctype|-s] the scorer type (default BLEU)"<<endl; cerr<<"[--scconfig|-c] configuration string passed to scorer"<<endl; cerr<<"\tThis is of the form NAME1:VAL1,NAME2:VAL2 etc "<<endl; cerr<<"[--reference|-R] comma separated list of reference files"<<endl; - cerr<<"[--candidate|-C] candidate file"<<endl; + cerr<<"[--candidate|-C] comma separated list of candidate files"<<endl; cerr<<"[--bootstrap|-b] number of booststraped samples (default 0 - no bootstraping)"<<endl; cerr<<"[--rseed|-r] the random seed for bootstraping (defaults to system clock)"<<endl; cerr<<"[--help|-h] print this message and exit"<<endl; + cerr<<endl; + cerr<<"Evaluator is able to compute more metrics at once. To do this,"<<endl; + cerr<<"separate scorers with semicolon (note that comma is used to separate"<<endl; + cerr<<"scorers in the interpolated scorer)."<<endl; + cerr<<endl; + cerr<<"If you specify only one metric and one candidate file, only the final score"<<endl; + cerr<<"will be printed to stdout. Otherwise each line will contain metric name"<<endl; + cerr<<"and/or filename and the final score. Since most of the metrics prints some"<<endl; + cerr<<"debuging info, consider redirecting stderr to /dev/null."<<endl; exit(1); } @@ -103,20 +113,32 @@ int main(int argc, char** argv) try { vector<string> refFiles; + vector<string> candFiles; vector<string> scorerTypes; + + if (reference.length() == 0) throw runtime_error("You have to specify at least one reference file."); split(reference,',',refFiles); if (candidate.length() == 0) throw runtime_error("You have to specify at least one candidate file."); - split(scorerType,';',scorerTypes); + split(candidate,',',candFiles); - for (vector<string>::const_iterator it = scorerTypes.begin(); it != scorerTypes.end(); ++it) + if (scorerType.length() == 0) throw runtime_error("You have to specify at least one scorer."); + split(scorerType,';',scorerTypes); + + if (candFiles.size() > 1) moreFiles = true; + if (scorerTypes.size() > 1) moreScorers = true; + + for (vector<string>::const_iterator fileIt = candFiles.begin(); fileIt != candFiles.end(); ++fileIt) { - scorer = ScorerFactory::getScorer(*it,scorerConfig); - scorer->setReferenceFiles(refFiles); - evaluate(candidate); - delete scorer; + for (vector<string>::const_iterator scorerIt = scorerTypes.begin(); scorerIt != scorerTypes.end(); ++scorerIt) + { + scorer = ScorerFactory::getScorer(*scorerIt,scorerConfig); + scorer->setReferenceFiles(refFiles); + evaluate(*fileIt); + delete scorer; + } } return EXIT_SUCCESS; @@ -167,9 +189,12 @@ void evaluate(const string& candFile) float avg = average(scores); float dev = stdDeviation(scores, avg); + if (moreFiles) cout << candFile << "\t"; + if (moreScorers) cout << scorer->getName() << "\t"; + cout.setf(ios::fixed,ios::floatfield); cout.precision(4); - cout << scorer->getName() << "\t" << avg << " [+-" << dev << "]"<< endl; + cout << avg << " [+-" << dev << "]"<< endl; } else { @@ -184,10 +209,13 @@ void evaluate(const string& candFile) candidates_t candidates(n, 0); float score = scorer->score(candidates); delete scoredata; - + + if (moreFiles) cout << candFile << "\t"; + if (moreScorers) cout << scorer->getName() << "\t"; + cout.setf(ios::fixed,ios::floatfield); cout.precision(4); - cout << scorer->getName() << "\t" << score << endl; + cout << score << endl; } } |