diff options
author | Matous Machacek <machacekmatous@gmail.com> | 2012-01-22 04:01:08 +0400 |
---|---|---|
committer | Matous Machacek <machacekmatous@gmail.com> | 2012-01-22 04:01:08 +0400 |
commit | b4a50ec50bea5d81effd42058b07013eafa6b963 (patch) | |
tree | ea0e82dd7d354f6b75a2a7c61ec6d51a5e41df13 /mert/evaluator.cpp | |
parent | 651fdba2455907699b67b0b7c91747909ec9e02e (diff) |
mert/evaluator can compute more metrics at once
Diffstat (limited to 'mert/evaluator.cpp')
-rw-r--r-- | mert/evaluator.cpp | 36 |
1 files changed, 14 insertions, 22 deletions
diff --git a/mert/evaluator.cpp b/mert/evaluator.cpp index c1d6039da..c5a59c427 100644 --- a/mert/evaluator.cpp +++ b/mert/evaluator.cpp @@ -22,14 +22,16 @@ float average(const vector<float>& list); float stdDeviation(const vector<float>& list, float avg); string int2string(int n); + + void usage() { - cerr<<"usage: evaluator [options] --reference ref1[,ref2[,ref3...]] --candidate cand1[,cand2[,cand3...]] "<<endl; - cerr<<"[--sctype|-s] the scorer type (default BLEU)"<<endl; + cerr<<"usage: evaluator [options] --reference ref1[,ref2[,ref3...]] --candidate candFile "<<endl; + cerr<<"[--sctype|-s] the scorer type (default BLEU). You can specify more scorers (separated by ;)"<<endl; cerr<<"[--scconfig|-c] configuration string passed to scorer"<<endl; cerr<<"\tThis is of the form NAME1:VAL1,NAME2:VAL2 etc "<<endl; cerr<<"[--reference|-R] comma separated list of reference files"<<endl; - cerr<<"[--candidate|-C] comma separated list of candidate files"<<endl; + cerr<<"[--candidate|-C] candidate file"<<endl; cerr<<"[--bootstrap|-b] number of booststraped samples (default 0 - no bootstraping)"<<endl; cerr<<"[--rseed|-r] the random seed for bootstraping (defaults to system clock)"<<endl; cerr<<"[--help|-h] print this message and exit"<<endl; @@ -101,30 +103,22 @@ int main(int argc, char** argv) try { vector<string> refFiles; - vector<string> candFiles; + vector<string> scorerTypes; if (reference.length() == 0) throw runtime_error("You have to specify at least one reference file."); split(reference,',',refFiles); if (candidate.length() == 0) throw runtime_error("You have to specify at least one candidate file."); - split(candidate,',',candFiles); - - scorer = ScorerFactory::getScorer(scorerType,scorerConfig); - cerr << "Using scorer: " << scorer->getName() << endl; - - scorer->setReferenceFiles(refFiles); - PrintUserTime("Reference files loaded"); + split(scorerType,';',scorerTypes); - - for (vector<string>::const_iterator it = candFiles.begin(); it != candFiles.end(); ++it) + for (vector<string>::const_iterator it = scorerTypes.begin(); it != scorerTypes.end(); ++it) { - evaluate(*it); + scorer = ScorerFactory::getScorer(*it,scorerConfig); + scorer->setReferenceFiles(refFiles); + evaluate(candidate); + delete scorer; } - PrintUserTime("Evaluation done"); - - delete scorer; - return EXIT_SUCCESS; } catch (const exception& e) { cerr << "Exception: " << e.what() << endl; @@ -149,8 +143,6 @@ void evaluate(const string& candFile) entries.push_back(scoreentry); } - PrintUserTime("Candidate file " + candFile + " loaded and stats prepared"); - int n = entries.size(); if (bootstrap) { @@ -177,7 +169,7 @@ void evaluate(const string& candFile) cout.setf(ios::fixed,ios::floatfield); cout.precision(4); - cout << "File: " << candFile << "\t" << scorer->getName() << " Average score: " << avg << "\tStandard deviation: " << dev << endl; + cout << scorer->getName() << "\t" << avg << " [+-" << dev << "]"<< endl; } else { @@ -195,7 +187,7 @@ void evaluate(const string& candFile) cout.setf(ios::fixed,ios::floatfield); cout.precision(4); - cout << "File: " << candFile << "\t" << scorer->getName() << " Score: " << score << endl; + cout << scorer->getName() << "\t" << score << endl; } } |