start integrating adam's suffix array

5 files changed, 52 insertions, 0 deletions

diff --git a/scripts/training/wrappers/adam-suffix-array/EVALUATION_europarl2007_decode.3 b/scripts/training/wrappers/adam-suffix-array/EVALUATION_europarl2007_decode.3
new file mode 100644
index 000000000..b8c2336ce
--- /dev/null
+++ b/scripts/training/wrappers/adam-suffix-array/EVALUATION_europarl2007_decode.3
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+PATH="/opt/local/bin:/opt/local/sbin:/opt/local/bin:/opt/local/sbin:/opt/local/bin:/opt/local/sbin:/opt/local/bin:/opt/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin:/usr/texbin"
+cd /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en
+echo 'starting at '`date`' on '`hostname`
+mkdir -p /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/evaluation
+
+/Users/hieuhoang/workspace/bin/current/dist/bin/moses_chart -threads 2 -cube-pruning-pop-limit 100 -s 30 -T /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/evaluation/europarl2007.output.3.trace -v 0 -f /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/evaluation/europarl2007.filtered.ini.3 < /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/evaluation/europarl2007.input.lc.3 > /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/evaluation/europarl2007.output.3
+
+echo 'finished at '`date`
+touch /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/steps/3/EVALUATION_europarl2007_decode.3.DONE
diff --git a/scripts/training/wrappers/adam-suffix-array/EVALUATION_newstest2007_decode.3 b/scripts/training/wrappers/adam-suffix-array/EVALUATION_newstest2007_decode.3
new file mode 100644
index 000000000..5a0598bb2
--- /dev/null
+++ b/scripts/training/wrappers/adam-suffix-array/EVALUATION_newstest2007_decode.3
@@ -0,0 +1,11 @@
+#!/bin/bash
+
+PATH="/opt/local/bin:/opt/local/sbin:/opt/local/bin:/opt/local/sbin:/opt/local/bin:/opt/local/sbin:/opt/local/bin:/opt/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin:/usr/texbin"
+cd /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en
+echo 'starting at '`date`' on '`hostname`
+mkdir -p /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/evaluation
+
+/Users/hieuhoang/workspace/bin/current/dist/bin/moses_chart -threads 2 -cube-pruning-pop-limit 100 -s 30 -T /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/evaluation/newstest2007.output.3.trace -v 0 -f /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/evaluation/newstest2007.filtered.ini.3 < /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/evaluation/newstest2007.input.lc.3 > /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/evaluation/newstest2007.output.3
+
+echo 'finished at '`date`
+touch /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/steps/3/EVALUATION_newstest2007_decode.3.DONE
diff --git a/scripts/training/wrappers/adam-suffix-array/run.sa.create.sh b/scripts/training/wrappers/adam-suffix-array/run.sa.create.sh
new file mode 100755
index 000000000..2fb5921bd
--- /dev/null
+++ b/scripts/training/wrappers/adam-suffix-array/run.sa.create.sh
@@ -0,0 +1,8 @@
+SA_EXEC_DIR=/Users/hieuhoang/workspace/github/cdec/sa-extract
+SOURCE_CORPUS=/Users/hieuhoang/workspace/data/europarl/exp/fr-en/training/corpus.2.fr
+TARGET_CORPUS=/Users/hieuhoang/workspace/data/europarl/exp/fr-en/training/corpus.2.en
+ALIGNMENT=/Users/hieuhoang/workspace/data/europarl/exp/fr-en/model/aligned.3.grow-diag-final-and
+SA_OUTPUT=/Users/hieuhoang/workspace/data/europarl/exp/fr-en/model/suffix-array.3
+
+~/workspace/bin/moses-smt/scripts/training/wrappers/suffix-array-create.sh $SA_EXEC_DIR $SOURCE_CORPUS $TARGET_CORPUS $ALIGNMENT $SA_OUTPUT
+
diff --git a/scripts/training/wrappers/adam-suffix-array/run.sa.extract.sh b/scripts/training/wrappers/adam-suffix-array/run.sa.extract.sh
new file mode 100755
index 000000000..425b19723
--- /dev/null
+++ b/scripts/training/wrappers/adam-suffix-array/run.sa.extract.sh
@@ -0,0 +1,8 @@
+SA_EXEC_DIR=/Users/hieuhoang/workspace/github/cdec/sa-extract
+MODEL_DIR=/Users/hieuhoang/workspace/data/europarl/exp/fr-en/model/suffix-array.3
+
+INPUT_FILE=/Users/hieuhoang/workspace/data/europarl/exp/fr-en/tuning/input.lc.2
+OUTPUT_DIR=/Users/hieuhoang/workspace/data/europarl/exp/fr-en/tuning/filtered.sa.3
+
+~/workspace/bin/moses-smt/scripts/training/wrappers/suffix-array-extract.sh $SA_EXEC_DIR $MODEL_DIR $INPUT_FILE $OUTPUT_DIR
+
diff --git a/scripts/training/wrappers/adam-suffix-array/run.sa.tune.sh b/scripts/training/wrappers/adam-suffix-array/run.sa.tune.sh
new file mode 100755
index 000000000..c832854bd
--- /dev/null
+++ b/scripts/training/wrappers/adam-suffix-array/run.sa.tune.sh
@@ -0,0 +1,14 @@
+#!/bin/bash
+
+PATH="/opt/local/bin:/opt/local/sbin:/opt/local/bin:/opt/local/sbin:/opt/local/bin:/opt/local/sbin:/opt/local/bin:/opt/local/sbin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin:/usr/X11/bin:/usr/texbin"
+cd /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en
+echo 'starting at '`date`' on '`hostname`
+mkdir -p /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/tuning
+
+/Users/hieuhoang/workspace/bin/current/scripts/training/mert-moses.pl /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/tuning/input.lc.2 /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/tuning/reference.lc.1 /Users/hieuhoang/workspace/bin/current/dist/bin/moses_chart /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/tuning/sa/moses.ini.3 --nbest 100 --working-dir /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/tuning/tmp.3  --decoder-flags "-threads 2 -cube-pruning-pop-limit 100 -s 30 -v 0 " --rootdir /Users/hieuhoang/workspace/bin/current/scripts -mertdir /Users/hieuhoang/workspace/bin/current/dist/bin --no-filter-phrase-table --continue
+
+mkdir -p /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/tuning
+cp /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/tuning/tmp.3/moses.ini /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/tuning/moses.ini.3
+
+echo 'finished at '`date`
+touch /Users/hieuhoang/unison/workspace/data/europarl/exp/fr-en/steps/3/TUNING_tune.3.DONE