diff options
author | Barry Haddow <barry.haddow@gmail.com> | 2012-06-01 19:27:59 +0400 |
---|---|---|
committer | Barry Haddow <barry.haddow@gmail.com> | 2012-06-01 19:27:59 +0400 |
commit | 47a6ee71cd89d85554990bce3d3c8813e26a2997 (patch) | |
tree | 0027ad821e9fe65bf80099d2d76b8927c6e13f49 | |
parent | 7a78e2d0f794cadaffe549bd278441e44e43e9a6 (diff) |
Update regression testing to work with new build layout.
Added the extract test to suite.
Disabled extract-rules test for now, since it fails.
Removed run-test-suite.perl, since tests are run with bjam.
-rw-r--r-- | regression-testing/Jamfile | 20 | ||||
-rwxr-xr-x | regression-testing/run-test-extract.perl | 2 | ||||
-rwxr-xr-x | regression-testing/run-test-suite.perl | 174 |
3 files changed, 17 insertions, 179 deletions
diff --git a/regression-testing/Jamfile b/regression-testing/Jamfile index 6afe1cf97..a7bd69506 100644 --- a/regression-testing/Jamfile +++ b/regression-testing/Jamfile @@ -21,12 +21,24 @@ if $(with-regtest) { actions reg_test_score { $(TOP)/regression-testing/run-test-scorer.perl --scorer=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<) } - reg_test score : [ glob $(TESTS)/score.* ] : ../scripts/training/phrase-extract//score : @reg_test_score ; + reg_test score : [ glob $(TESTS)/score.* ] : ../phrase-extract//score : @reg_test_score ; + + actions reg_test_extract { + $(TOP)/regression-testing/run-test-extract.perl --extractor=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<) + } + reg_test extract : [ glob $(TESTS)/extract.* ] : ../phrase-extract//extract : @reg_test_extract ; + + + actions reg_test_extractrules { + $(TOP)/regression-testing/run-test-extract.perl --extractor=$(>) --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<) + } + reg_test extractrules : [ glob $(TESTS)/extract-rules.* : $(TESTS)/extract-rules.hierarchical ] : ../phrase-extract//extract-rules : @reg_test_extractrules ; + actions reg_test_mert { - $(TOP)/regression-testing/run-test-mert.perl --mert-dir=$(TOP)/mert --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<) + $(TOP)/regression-testing/run-test-mert.perl --test=$(<:B) --data-dir=$(with-regtest) --test-dir=$(TESTS) && touch $(<) } - reg_test mert : [ glob $(TESTS)/mert.* ] : ../mert//legacy : @reg_test_mert ; + reg_test mert : [ glob $(TESTS)/mert.* ] : ../mert//mert : @reg_test_mert ; - alias all : phrase chart mert score ; + alias all : phrase chart mert score extract extractrules ; } diff --git a/regression-testing/run-test-extract.perl b/regression-testing/run-test-extract.perl index c53e6247d..d13ef62d6 100755 --- a/regression-testing/run-test-extract.perl +++ b/regression-testing/run-test-extract.perl @@ -53,7 +53,7 @@ my $truthPath = "$test_dir/$test_name/truth/"; if (-e $outPath) { - my $cmd = "diff --exclude=.DS_Store $outPath/ $truthPath/ | wc -l"; + my $cmd = "diff --exclude=.DS_Store --exclude=._* $outPath/ $truthPath/ | wc -l"; my $numDiff = `$cmd`; if ($numDiff == 0) diff --git a/regression-testing/run-test-suite.perl b/regression-testing/run-test-suite.perl deleted file mode 100755 index f09f4b31f..000000000 --- a/regression-testing/run-test-suite.perl +++ /dev/null @@ -1,174 +0,0 @@ -#!/usr/bin/perl -w - -# $Id$ - -use strict; -use FindBin qw($Bin); - -my $script_dir; BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, $script_dir; } -use Getopt::Long; - -############################################################ -my @tests = qw ( - extract.phrase-based - extract-rules.hierarchical - score.phrase-based - score.phrase-based-inv - score.phrase-based-with-alignment - score.phrase-based-with-alignment-inv - score.hierarchical - score.hierarchical-inv - mert.basic - mert.pro - mert.extractor-txt - mert.extractor-bin - chart.target-syntax - chart.target-syntax.ondisk - chart.hierarchical - chart.hierarchical-withsrilm - chart.hierarchical.ondisk - phrase.basic-surface-only - phrase.basic-surface-only-withirstlm - phrase.basic-surface-only-withirstlm-binlm - phrase.basic-lm-oov - phrase.ptable-filtering - phrase.multi-factor - phrase.multi-factor-drop - phrase.confusionNet-surface-only - phrase.confusionNet-multi-factor - phrase.basic-surface-binptable - phrase.multi-factor-binptable - phrase.nbest-multi-factor - phrase.lattice-surface - phrase.lattice-distortion - phrase.lexicalized-reordering - phrase.lexicalized-reordering-bin - phrase.lexicalized-reordering-cn - phrase.consensus-decoding-surface - phrase.continue-partial-translation - phrase.show-weights.lex-reorder - phrase.show-weights - phrase.xml-markup -); - #phrase.basic-lm-oov-withkenlm - #phrase.basic-surface-only-withkenlm - #phrase.basic-surface-only-withkenlm.bin - #chart.hierarchical-withkenlm - -############################################################ -use MosesRegressionTesting; -use File::Temp qw ( tempfile ); -use POSIX qw ( strftime ); - -my $decoderPhrase = "$Bin/../bin/moses"; -my $decoderChart = "$Bin/../bin/moses_chart"; -my $scoreExe = "$Bin/../bin/score"; -my $extractorExe = "$Bin/../bin/extract"; -my $extractorRulesExe = "$Bin/../bin/extract-rules"; -my $kenlmBinarizer = "$Bin/../bin/build_binary"; -my $test_dir; -my $BIN_TEST = $script_dir; -my $data_dir; - -GetOptions("data-dir=s" => \$data_dir, - ) or exit 1; - - -$data_dir = MosesRegressionTesting::find_data_directory($BIN_TEST, $data_dir); - -my $test_run = "$BIN_TEST/run-single-test.pl --data-dir=$data_dir"; -$test_dir = "$data_dir/tests"; -$test_run .= " --test-dir=$test_dir" if $test_dir; - -print "Data directory: $data_dir\n"; - -print "Running tests: @tests\n\n"; - -print "TEST NAME STATUS PATH TO RESULTS\n"; -my $lb = "---------------------------------------------------------------------------------------------------------\n"; -print $lb; - -my $fail = 0; -my @failed; -foreach my $test (@tests) -{ - my $cmd; - my @tokens = split('\.', $test); - my $model_type = $tokens[0]; - - if ($model_type eq 'phrase') - { - $cmd .= "$BIN_TEST/run-single-test.perl $test_run --decoder=$decoderPhrase"; - } - elsif ($model_type eq 'chart') - { - $cmd .= "$BIN_TEST/run-single-test.perl $test_run --decoder=$decoderChart"; - } - elsif ($model_type eq 'score') - { - $cmd .= "$BIN_TEST/run-test-scorer.perl $test_run --scorer=$scoreExe"; - } - elsif ($model_type eq 'extract') - { - $cmd .= "$BIN_TEST/run-test-extract.perl $test_run --extractor=$extractorExe"; - } - elsif ($model_type eq 'extract-rules') - { - $cmd .= "$BIN_TEST/run-test-extract.perl $test_run --extractor=$extractorRulesExe"; - } - elsif ($model_type eq "mert") - { - $cmd .= "$BIN_TEST/run-test-mert.perl $test_run"; - } - elsif ($model_type eq "kenlmbin") - { - $cmd .= "$BIN_TEST/run-kenlm-binarizer.perl --binarizer=$kenlmBinarizer"; - } - else - { - print "FAIL"; - } - - $cmd .= " --test=$test"; - -print STDERR "cmd = $cmd\n"; - - my ($res, $output, $results_path) = do_test($cmd); - format STDOUT = -@<<<<<<<<<<<<<<<<<<<<<< @<<<<<<<<< @<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<<< -$test, $res, $results_path -. - write; - if ($res eq 'FAIL') { - print "$lb$output$lb"; - $fail++; - push @failed, $test; - } else { -# TOTAL_WALLTIME result=BASELINE=11, TEST=12 DELTA=1 PCT CHANGE=9.09 - if ($output =~ /TOTAL_WALLTIME\s+result\s*=\s*([^\n]+)/o) { - print "\t\tTiming statistics: $1\n"; - } - } -} - -my $total = scalar @tests; -my $fail_percentage = int(100 * $fail / $total); -my $pass_percentage = int(100 * ($total-$fail) / $total); -print "\n$pass_percentage% of the tests passed.\n"; -print "$fail_percentage% of the tests failed.\n"; -if ($fail_percentage>0) { print "\nPLEASE INVESTIGATE THESE FAILED TESTS: @failed\n"; } -exit 2 if $fail > 0; - -sub do_test { - my ($test) = @_; - my $o = `$test 2>&1`; - my $res = 'PASS'; - $res = 'FAIL' if ($? > 0); - my $od = ''; - if ($o =~ /RESULTS AVAILABLE IN: (.*)$/m) { - $od = $1; - $o =~ s/^RESULTS AVAIL.*$//mo; - } - return ($res, $o, $od); -} - |