diff options
author | Matous Machacek <machacekmatous@gmail.com> | 2012-02-27 02:04:02 +0400 |
---|---|---|
committer | Matous Machacek <machacekmatous@gmail.com> | 2012-02-27 02:04:02 +0400 |
commit | e3f0280f274a92a75c98f2243c0a62f79dd39683 (patch) | |
tree | 6bb7be35bed9fb4c2507d9bfb4a3707e6ac0cad1 /mert/evaluator.cpp | |
parent | 99a98a336bc1382a95375c0bc688a0cba4d11c27 (diff) |
Change of evaluator usage (see mert/evaluator --help).
Diffstat (limited to 'mert/evaluator.cpp')
-rw-r--r-- | mert/evaluator.cpp | 47 |
1 files changed, 30 insertions, 17 deletions
diff --git a/mert/evaluator.cpp b/mert/evaluator.cpp index 2fcda0140..21556e9ee 100644 --- a/mert/evaluator.cpp +++ b/mert/evaluator.cpp @@ -138,10 +138,19 @@ void usage() cerr << "[--help|-h] print this message and exit" << endl; cerr << endl; cerr << "Evaluator is able to compute more metrics at once. To do this," << endl; - cerr << "separate scorers with semicolon (note that comma is used to separate" << endl; - cerr << "scorers in the interpolated scorer)." << endl; + cerr << "specify more --sctype arguments. You can also specify more --scconfig strings." << endl; cerr << endl; - cerr << "If you specify only one metric and one candidate file, only the final score" << endl; + cerr << "The example below prints BLEU score, PER score and interpolated" << endl; + cerr << "score of CDER and PER with the given weights." << endl; + cerr << endl; + cerr << "./evaluator \\" << endl; + cerr << "\t--sctype BLEU --scconfig reflen:closest \\" << endl; + cerr << "\t--sctype PER \\" << endl; + cerr << "\t--sctype CDER,PER --scconfig weights:0.25+0.75 \\" << endl; + cerr << "\t--candidate CANDIDATE \\" << endl; + cerr << "\t--reference REFERENCE" << endl; + cerr << endl; + cerr << "If you specify only one scorer and one candidate file, only the final score" << endl; cerr << "will be printed to stdout. Otherwise each line will contain metric name" << endl; cerr << "and/or filename and the final score. Since most of the metrics prints some" << endl; cerr << "debuging info, consider redirecting stderr to /dev/null." << endl; @@ -161,8 +170,8 @@ static struct option long_options[] = { // Options used in evaluator. struct ProgramOption { - string scorer_type; - string scorer_config; + vector<string> scorer_types; + vector<string> scorer_configs; string reference; string candidate; int bootstrap; @@ -170,9 +179,7 @@ struct ProgramOption { bool has_seed; ProgramOption() - : scorer_type("BLEU"), - scorer_config(""), - reference(""), + : reference(""), candidate(""), bootstrap(0), seed(0), @@ -182,13 +189,16 @@ struct ProgramOption { void ParseCommandOptions(int argc, char** argv, ProgramOption* opt) { int c; int option_index; + int last_scorer_index = -1; while ((c = getopt_long(argc, argv, "s:c:R:C:b:r:h", long_options, &option_index)) != -1) { switch(c) { case 's': - opt->scorer_type = string(optarg); + opt->scorer_types.push_back(string(optarg)); + opt->scorer_configs.push_back(string("")); + last_scorer_index++; break; case 'c': - opt->scorer_config = string(optarg); + opt->scorer_configs[last_scorer_index] = string(optarg); break; case 'R': opt->reference = string(optarg); @@ -207,6 +217,13 @@ void ParseCommandOptions(int argc, char** argv, ProgramOption* opt) { usage(); } } + + // Add default scorer if no scorer provided + if (opt->scorer_types.size() == 0) + { + opt->scorer_types.push_back(string("BLEU")); + opt->scorer_configs.push_back(string("")); + } } void InitSeed(const ProgramOption *opt) { @@ -236,7 +253,6 @@ int main(int argc, char** argv) try { vector<string> refFiles; vector<string> candFiles; - vector<string> scorerTypes; if (option.reference.length() == 0) throw runtime_error("You have to specify at least one reference file."); split(option.reference, ',', refFiles); @@ -244,17 +260,14 @@ int main(int argc, char** argv) if (option.candidate.length() == 0) throw runtime_error("You have to specify at least one candidate file."); split(option.candidate, ',', candFiles); - if (option.scorer_type.length() == 0) throw runtime_error("You have to specify at least one scorer."); - split(option.scorer_type, ';', scorerTypes); - if (candFiles.size() > 1) g_has_more_files = true; - if (scorerTypes.size() > 1) g_has_more_scorers = true; + if (option.scorer_types.size() > 1) g_has_more_scorers = true; for (vector<string>::const_iterator fileIt = candFiles.begin(); fileIt != candFiles.end(); ++fileIt) { - for (vector<string>::const_iterator scorerIt = scorerTypes.begin(); scorerIt != scorerTypes.end(); ++scorerIt) + for (size_t i = 0; i < option.scorer_types.size(); i++) { - g_scorer = ScorerFactory::getScorer(*scorerIt, option.scorer_config); + g_scorer = ScorerFactory::getScorer(option.scorer_types[i], option.scorer_configs[i]); g_scorer->setReferenceFiles(refFiles); EvaluatorUtil::evaluate(*fileIt, option.bootstrap); delete g_scorer; |