diff options
author | hieuhoang1972 <hieuhoang1972@1f5c12ca-751b-0410-a591-d2e778427230> | 2011-09-30 18:12:18 +0400 |
---|---|---|
committer | hieuhoang1972 <hieuhoang1972@1f5c12ca-751b-0410-a591-d2e778427230> | 2011-09-30 18:12:18 +0400 |
commit | f51239cf68950f05eda8841432fceac3d0a82cf3 (patch) | |
tree | 74252e3d7fdda53ac43141a7f3ac9623d41387fc /regression-testing | |
parent | 521d7b219900b233a143cd45aa0593daf4a0a8b0 (diff) |
kenlm regression tests
git-svn-id: https://mosesdecoder.svn.sourceforge.net/svnroot/mosesdecoder/trunk@4284 1f5c12ca-751b-0410-a591-d2e778427230
Diffstat (limited to 'regression-testing')
22 files changed, 416 insertions, 1 deletions
diff --git a/regression-testing/MosesRegressionTesting.pm b/regression-testing/MosesRegressionTesting.pm index 77d6e730c..5a5b977ba 100644 --- a/regression-testing/MosesRegressionTesting.pm +++ b/regression-testing/MosesRegressionTesting.pm @@ -5,7 +5,7 @@ use strict; # if your tests need a new version of the test data, increment this # and make sure that a moses-regression-tests-vX.Y is available for # download from statmt.org (redpony AT umd dot edu for more info) -use constant TESTING_DATA_VERSION => '7'; +use constant TESTING_DATA_VERSION => '8'; # find the data directory in a few likely locations and make sure # that it is the correct version diff --git a/regression-testing/run-test-suite.perl b/regression-testing/run-test-suite.perl index 2eabce02c..8a05e15bb 100755 --- a/regression-testing/run-test-suite.perl +++ b/regression-testing/run-test-suite.perl @@ -23,10 +23,15 @@ my @tests = qw ( chart.target-syntax.ondisk chart.hierarchical chart.hierarchical-withsrilm + chart.hierarchical-withkenlm chart.hierarchical.ondisk phrase.basic-surface-only phrase.basic-surface-only-withirstlm phrase.basic-surface-only-withirstlm-binlm + phrase.basic-surface-only-withkenlm + phrase.basic-surface-only-withkenlm.bin + phrase.basic-lm-oov + phrase.basic-lm-oov-withkenlm phrase.ptable-filtering phrase.multi-factor phrase.multi-factor-drop diff --git a/regression-testing/tests/chart.hierarchical-withkenlm/filter-stderr.pl b/regression-testing/tests/chart.hierarchical-withkenlm/filter-stderr.pl new file mode 100755 index 000000000..2f6e17607 --- /dev/null +++ b/regression-testing/tests/chart.hierarchical-withkenlm/filter-stderr.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl + +BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; } +use RegTestUtils; + +$x=0; +while (<>) { + chomp; + + if (/^Finished loading LanguageModels/) { + my $time = RegTestUtils::readTime($_); + print "LMLOAD_TIME ~ $time\n"; + } + if (/^Finished loading phrase tables/) { + my $time = RegTestUtils::readTime($_); + print "PTLOAD_TIME ~ $time\n"; + } + next unless /^BEST TRANSLATION:/; + my $pscore = RegTestUtils::readHypoScore($_); + $x++; + print "SCORE_$x = $pscore\n"; +} diff --git a/regression-testing/tests/chart.hierarchical-withkenlm/filter-stdout.pl b/regression-testing/tests/chart.hierarchical-withkenlm/filter-stdout.pl new file mode 100755 index 000000000..476ddf6e9 --- /dev/null +++ b/regression-testing/tests/chart.hierarchical-withkenlm/filter-stdout.pl @@ -0,0 +1,7 @@ +#!/usr/bin/perl +$x=0; +while (<>) { + chomp; + $x++; + print "TRANSLATION_$x=$_\n"; +} diff --git a/regression-testing/tests/chart.hierarchical-withkenlm/moses.ini b/regression-testing/tests/chart.hierarchical-withkenlm/moses.ini new file mode 100644 index 000000000..d8c7694fb --- /dev/null +++ b/regression-testing/tests/chart.hierarchical-withkenlm/moses.ini @@ -0,0 +1,69 @@ +######################### +### MOSES CONFIG FILE ### +######################### + +# input factors +[input-factors] +0 + +# mapping steps +[mapping] +0 T 0 +1 T 1 + +# translation tables: source-factors, target-factors, number of scores, file +[ttable-file] +6 0 0 5 ${MODEL_PATH}/hierarchical/phrase-table.0-0.1 +6 0 0 1 ${MODEL_PATH}/hierarchical/phrase-table.0-0.2 + +# no generation models, no generation-file section + +# language models: type(srilm/irstlm), factors, order, file +[lmodel-file] +8 0 3 ${LM_PATH}/news.de.kenlm + + +# limit on how many phrase translations e for each phrase f are loaded +# 0 = all elements loaded +[ttable-limit] +20 + + +# language model weights +[weight-l] +0.273687 + + +# translation model weights +[weight-t] +0.077197 +0.014533 +0.059406 +0.001891 +-0.098314 +0.131945 + +# no generation models, no weight-generation section + +# word penalty +[weight-w] +-0.343026 + +[cube-pruning-pop-limit] +1000 + +[non-terminals] +X + +[search-algorithm] +3 + +[inputtype] +3 + +[max-chart-span] +20 +1000 + +[weight-d] +1 diff --git a/regression-testing/tests/chart.hierarchical-withkenlm/to-translate.txt b/regression-testing/tests/chart.hierarchical-withkenlm/to-translate.txt new file mode 100644 index 000000000..142ac43e0 --- /dev/null +++ b/regression-testing/tests/chart.hierarchical-withkenlm/to-translate.txt @@ -0,0 +1,5 @@ +Prague Stock Market falls to minus by the end of the trading day +after a sharp drop in the morning , the Prague Stock Market corrected its losses . +transactions with stocks from the Czech Energy Enterprise ( ÄŒEZ ) reached nearly half of the regular daily trading . +the Prague Stock Market immediately continued its fall from Monday at the beginning of Tuesday 's trading , when it dropped by nearly six percent . +this time the fall in stocks on Wall Street is responsible for the drop . diff --git a/regression-testing/tests/chart.hierarchical-withkenlm/truth/results.txt b/regression-testing/tests/chart.hierarchical-withkenlm/truth/results.txt new file mode 100644 index 000000000..c77f070ce --- /dev/null +++ b/regression-testing/tests/chart.hierarchical-withkenlm/truth/results.txt @@ -0,0 +1,15 @@ +TRANSLATION_1=Prag Stock Markt fällt , durch das Ende der Handel Tag +TRANSLATION_2=nach einem steilen Rückgang der Morgen , dem Prager Stock Markt korrigiert ihre Verluste . +TRANSLATION_3=Transaktionen mit Aktien von der tschechischen Energy Enterprise ( ÄŒEZ ) erreicht beinahe die Hälfte der regelmäßigen täglich Handel . +TRANSLATION_4=der Prager Stock Markt sofort ihren Sturz von Montag zu Beginn der Handel Tuesday , wenn es um beinahe sechs Prozent . +TRANSLATION_5=dieses Mal den Rückgang der Aktien an der Wall Street verantwortlich ist für die fallen . +LMLOAD_TIME ~ 1.000 +PTLOAD_TIME ~ 1.000 +PTLOAD_TIME ~ 83.000 +PTLOAD_TIME ~ 83.000 +SCORE_1 = -12.100 +SCORE_2 = -12.344 +SCORE_3 = -116.629 +SCORE_4 = -121.386 +SCORE_5 = -8.128 +TOTAL_WALLTIME ~ 105 diff --git a/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stderr.pl b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stderr.pl new file mode 100755 index 000000000..2f6e17607 --- /dev/null +++ b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stderr.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl + +BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; } +use RegTestUtils; + +$x=0; +while (<>) { + chomp; + + if (/^Finished loading LanguageModels/) { + my $time = RegTestUtils::readTime($_); + print "LMLOAD_TIME ~ $time\n"; + } + if (/^Finished loading phrase tables/) { + my $time = RegTestUtils::readTime($_); + print "PTLOAD_TIME ~ $time\n"; + } + next unless /^BEST TRANSLATION:/; + my $pscore = RegTestUtils::readHypoScore($_); + $x++; + print "SCORE_$x = $pscore\n"; +} diff --git a/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stdout.pl b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stdout.pl new file mode 100755 index 000000000..476ddf6e9 --- /dev/null +++ b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/filter-stdout.pl @@ -0,0 +1,7 @@ +#!/usr/bin/perl +$x=0; +while (<>) { + chomp; + $x++; + print "TRANSLATION_$x=$_\n"; +} diff --git a/regression-testing/tests/phrase.basic-lm-oov-withkenlm/moses.ini b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/moses.ini new file mode 100644 index 000000000..3c2558505 --- /dev/null +++ b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/moses.ini @@ -0,0 +1,53 @@ +# moses.ini for regression test + +[ttable-file] +0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz + +# language model +[lmodel-file] +8 0 3 ${LM_PATH}/europarl.en.srilm.gz +# limit on how many phrase translations e for each phrase f are loaded +[ttable-limit] +#ttable element load limit 0 = all elements loaded +20 + +# distortion (reordering) weight +[weight-d] +0.141806519223522 + +# language model weight +[weight-l] +0.142658800199951 +0.1 + +# translation model weight (phrase translation, lexical weighting) +[weight-t] +0.00402447059454402 +0.0685647475075862 +0.294089113124688 +0.0328320356515851 +-0.0426081987467227 + +# word penalty +[weight-w] +-0.273416114951401 + +[distortion-limit] +4 + +[beam-threshold] +0.03 + +[input-factors] +0 + +[mapping] +T 0 + +[lmodel-oov-feature] +1 + + +[verbose] +2 + diff --git a/regression-testing/tests/phrase.basic-lm-oov-withkenlm/to-translate.txt b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/to-translate.txt new file mode 100644 index 000000000..a505702d1 --- /dev/null +++ b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/to-translate.txt @@ -0,0 +1,5 @@ +ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ? +schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind . +vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen . +das parlament will das auf zweierlei weise tun . +nur dann werden die europäischen institutionen auch ihrem auftrag gerecht . diff --git a/regression-testing/tests/phrase.basic-lm-oov-withkenlm/truth/results.txt b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/truth/results.txt new file mode 100644 index 000000000..974429d61 --- /dev/null +++ b/regression-testing/tests/phrase.basic-lm-oov-withkenlm/truth/results.txt @@ -0,0 +1,13 @@ +TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? +TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . +TRANSLATION_3=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . +TRANSLATION_4=parliament wants the in two ways . +TRANSLATION_5=only then will the european institutions to its mandate . +LMLOAD_TIME ~ 8.00 +PTLOAD_TIME ~ 9.00 +SCORE_1 = -14.843 +SCORE_2 = -152.932 +SCORE_3 = -277.957 +SCORE_4 = -5.995 +SCORE_5 = -7.015 +TOTAL_WALLTIME ~ 28 diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stderr.pl b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stderr.pl new file mode 100755 index 000000000..2f6e17607 --- /dev/null +++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stderr.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl + +BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; } +use RegTestUtils; + +$x=0; +while (<>) { + chomp; + + if (/^Finished loading LanguageModels/) { + my $time = RegTestUtils::readTime($_); + print "LMLOAD_TIME ~ $time\n"; + } + if (/^Finished loading phrase tables/) { + my $time = RegTestUtils::readTime($_); + print "PTLOAD_TIME ~ $time\n"; + } + next unless /^BEST TRANSLATION:/; + my $pscore = RegTestUtils::readHypoScore($_); + $x++; + print "SCORE_$x = $pscore\n"; +} diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stdout.pl b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stdout.pl new file mode 100755 index 000000000..476ddf6e9 --- /dev/null +++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/filter-stdout.pl @@ -0,0 +1,7 @@ +#!/usr/bin/perl +$x=0; +while (<>) { + chomp; + $x++; + print "TRANSLATION_$x=$_\n"; +} diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/moses.ini b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/moses.ini new file mode 100644 index 000000000..25bd27173 --- /dev/null +++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/moses.ini @@ -0,0 +1,49 @@ +# moses.ini for regression test + +[ttable-file] +0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz + +# language model +[lmodel-file] +8 0 3 ${LM_PATH}/europarl.en.srilm.gz +# limit on how many phrase translations e for each phrase f are loaded +[ttable-limit] +#ttable element load limit 0 = all elements loaded +20 + +# distortion (reordering) weight +[weight-d] +0.141806519223522 + +# language model weight +[weight-l] +0.142658800199951 + +# translation model weight (phrase translation, lexical weighting) +[weight-t] +0.00402447059454402 +0.0685647475075862 +0.294089113124688 +0.0328320356515851 +-0.0426081987467227 + +# word penalty +[weight-w] +-0.273416114951401 + +[distortion-limit] +4 + +[beam-threshold] +0.03 + +[input-factors] +0 + +[mapping] +T 0 + + +[verbose] +2 + diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/to-translate.txt b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/to-translate.txt new file mode 100644 index 000000000..a505702d1 --- /dev/null +++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/to-translate.txt @@ -0,0 +1,5 @@ +ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ? +schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind . +vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen . +das parlament will das auf zweierlei weise tun . +nur dann werden die europäischen institutionen auch ihrem auftrag gerecht . diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/truth/results.txt b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/truth/results.txt new file mode 100644 index 000000000..44a0502c8 --- /dev/null +++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm.bin/truth/results.txt @@ -0,0 +1,13 @@ +TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? +TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . +TRANSLATION_3=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . +TRANSLATION_4=parliament wants the in two ways . +TRANSLATION_5=only then will the european institutions to its mandate . +LMLOAD_TIME ~ 8.00 +PTLOAD_TIME ~ 9.00 +SCORE_1 = -14.843 +SCORE_2 = -153.032 +SCORE_3 = -278.157 +SCORE_4 = -5.995 +SCORE_5 = -7.015 +TOTAL_WALLTIME ~ 28 diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stderr.pl b/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stderr.pl new file mode 100755 index 000000000..2f6e17607 --- /dev/null +++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stderr.pl @@ -0,0 +1,22 @@ +#!/usr/bin/perl + +BEGIN { use Cwd qw/ abs_path /; use File::Basename; $script_dir = dirname(abs_path($0)); push @INC, "$script_dir/../perllib"; } +use RegTestUtils; + +$x=0; +while (<>) { + chomp; + + if (/^Finished loading LanguageModels/) { + my $time = RegTestUtils::readTime($_); + print "LMLOAD_TIME ~ $time\n"; + } + if (/^Finished loading phrase tables/) { + my $time = RegTestUtils::readTime($_); + print "PTLOAD_TIME ~ $time\n"; + } + next unless /^BEST TRANSLATION:/; + my $pscore = RegTestUtils::readHypoScore($_); + $x++; + print "SCORE_$x = $pscore\n"; +} diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stdout.pl b/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stdout.pl new file mode 100755 index 000000000..476ddf6e9 --- /dev/null +++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm/filter-stdout.pl @@ -0,0 +1,7 @@ +#!/usr/bin/perl +$x=0; +while (<>) { + chomp; + $x++; + print "TRANSLATION_$x=$_\n"; +} diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm/moses.ini b/regression-testing/tests/phrase.basic-surface-only-withkenlm/moses.ini new file mode 100644 index 000000000..25bd27173 --- /dev/null +++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm/moses.ini @@ -0,0 +1,49 @@ +# moses.ini for regression test + +[ttable-file] +0 0 0 5 ${MODEL_PATH}/basic-surface-only/phrase-table.gz + +# language model +[lmodel-file] +8 0 3 ${LM_PATH}/europarl.en.srilm.gz +# limit on how many phrase translations e for each phrase f are loaded +[ttable-limit] +#ttable element load limit 0 = all elements loaded +20 + +# distortion (reordering) weight +[weight-d] +0.141806519223522 + +# language model weight +[weight-l] +0.142658800199951 + +# translation model weight (phrase translation, lexical weighting) +[weight-t] +0.00402447059454402 +0.0685647475075862 +0.294089113124688 +0.0328320356515851 +-0.0426081987467227 + +# word penalty +[weight-w] +-0.273416114951401 + +[distortion-limit] +4 + +[beam-threshold] +0.03 + +[input-factors] +0 + +[mapping] +T 0 + + +[verbose] +2 + diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm/to-translate.txt b/regression-testing/tests/phrase.basic-surface-only-withkenlm/to-translate.txt new file mode 100644 index 000000000..a505702d1 --- /dev/null +++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm/to-translate.txt @@ -0,0 +1,5 @@ +ich frage sie also , herr präsident : stellen die unterschiedlichen arbeitskosten somit nicht auch eine beschränkung des freien wettbewerbs in der europäischen union dar ? +schaut man sich die fälligkeitspläne der ausführung des haushalts für die rubriken 2 , 3 , 4 und 7 an , stellt man fest , dass nur durchschnittlich 8 % aller verpflichtungen durch zahlungen gedeckt sind . +vor drei jahren haben wir mit unserer beschäftigungsinitiative begonnen , indem wir kleinen und mittleren unternehmen halfen , chancenkapital zu bekommen . +das parlament will das auf zweierlei weise tun . +nur dann werden die europäischen institutionen auch ihrem auftrag gerecht . diff --git a/regression-testing/tests/phrase.basic-surface-only-withkenlm/truth/results.txt b/regression-testing/tests/phrase.basic-surface-only-withkenlm/truth/results.txt new file mode 100644 index 000000000..44a0502c8 --- /dev/null +++ b/regression-testing/tests/phrase.basic-surface-only-withkenlm/truth/results.txt @@ -0,0 +1,13 @@ +TRANSLATION_1=i ask you , therefore , mr president , the different labour costs are therefore not a restriction of free competition in the european union ? +TRANSLATION_2=if we look at the fälligkeitspläne the implementation of the budget for the categories 2 , 3 , 4 and 7 to , we see that only an average of 8 % of commitments by payments are met . +TRANSLATION_3=three years ago our employment strategy , we started by small and medium-sized enterprises , halfen chancenkapital to obtain . +TRANSLATION_4=parliament wants the in two ways . +TRANSLATION_5=only then will the european institutions to its mandate . +LMLOAD_TIME ~ 8.00 +PTLOAD_TIME ~ 9.00 +SCORE_1 = -14.843 +SCORE_2 = -153.032 +SCORE_3 = -278.157 +SCORE_4 = -5.995 +SCORE_5 = -7.015 +TOTAL_WALLTIME ~ 28 |