diff options
author | Kenneth Heafield <github@kheafield.com> | 2011-11-25 19:33:46 +0400 |
---|---|---|
committer | Kenneth Heafield <github@kheafield.com> | 2011-11-25 19:33:46 +0400 |
commit | 8100d587c7b282943ad3ab3798e665db55d14243 (patch) | |
tree | 240efddc05b059510091a1812388ec82dfd97ac7 /scripts | |
parent | 67c79de6f9b17af23353d930ac97b98ef10749c2 (diff) |
Replace scripts build system.
Diffstat (limited to 'scripts')
-rw-r--r-- | scripts/Jamfile | 32 | ||||
-rw-r--r-- | scripts/Makefile | 185 | ||||
-rw-r--r-- | scripts/README | 2 | ||||
-rwxr-xr-x | scripts/check-dependencies.pl | 41 | ||||
-rw-r--r-- | scripts/released-files | 107 | ||||
-rw-r--r-- | scripts/training/Jamfile | 2 |
6 files changed, 31 insertions, 338 deletions
diff --git a/scripts/Jamfile b/scripts/Jamfile index 72bf0402e..c20827bd8 100644 --- a/scripts/Jamfile +++ b/scripts/Jamfile @@ -1,14 +1,38 @@ +#MOSES SCRIPTS INSTRUCTIONS +# +#--bindir=/path/to/bindir where GIZA++ binaries live +# +#--install=/path/to/install to set a path to install (default dist) +#--git appends the git revision to the install directory +# +#--with-boost=/path/to/boost to specify a non-standard Boost installation +# +#See also: bjam --help one level up. + import option ; build-project training ; location = [ option.get "install" : $(TOP)/scripts/dist ] ; +location = $(location)$(GITTAG) ; -constant BINDIR : [ option.get "bindir" ] ; - -if ! $(BINDIR) { - exit "Pass --bindir=/path/to/bindir to indicate where GIZA++, snt2cooc.out, and mkcls live." : 1 ; +bindir = [ option.get "bindir" ] ; +if ! $(bindir) { + echo "Pass --bindir=/path/to/bindir" ; + echo "It should contain GIZA++, snt2cooc.out, and mkcls." ; + echo "These are available from http://www.fjoch.com/GIZA++.html and" ; + echo "http://www-i6.informatik.rwth-aachen.de/Colleagues/och/software/mkcls.html ." ; + exit : 1 ; +} +rule check-for-bin ( name ) { + if ! [ FILE_OPEN $(bindir)/$(name) : "r" ] { + exit "Did not find $(bindir)/$(name)." : 1 ; + } } +check-for-bin GIZA++ ; +check-for-bin snt2cooc.out ; +check-for-bin mkcls ; +constant BINDIR : $(bindir) ; #These two used to live in a tools directory. install ghkm : training/phrase-extract/extract-ghkm//extract-ghkm : <location>$(location)/training/phrase-extract/extract-ghkm/tools ; diff --git a/scripts/Makefile b/scripts/Makefile deleted file mode 100644 index 084172a0b..000000000 --- a/scripts/Makefile +++ /dev/null @@ -1,185 +0,0 @@ -# This makefile is here to simplify the automatic releases (and tests!!!) -# of the scripts - - -TS?=$(shell date '+%Y%m%d-%H%M') -DS?=$(shell date '+%Y%m%d') - -# Set TARGETDIR to directory where you want the compiled scripts to be copied -# to. -# Set BINDIR to the directory where GIZA++ and other tools are installed. -TARGETDIR=/home/bhaddow/moses.github -BINDIR=/home/bhaddow/moses/bin - -MAIN_SCRIPTS_TARGET_DIR=$(TARGETDIR) -# MAIN_SCRIPTS_TARGET_DIR=$(shell echo `pwd`/temp) - -RELEASEDIR=$(TARGETDIR)/scripts-$(TS) -# RELEASEDIR=$(shell echo `pwd`/temp) - - - -## Rules to compile parts that need compilation - - -all: compile - -SUBDIRS=training/phrase-extract training/symal training/lexical-reordering ems/biconcor -SUBDIRS_CLEAN=$(SUBDIRS) training/compact-rule-table - -compile: compile-compact-rule-table compile-extract-ghkm - touch release-exclude # No files excluded by default - pwd=`pwd`; \ - for subdir in $(SUBDIRS); do \ - $(MAKE) -C $$subdir || exit 1; \ - echo "### Compiler $$subdir"; \ - cd $$pwd; \ - done - ## All files that need compilation were compiled - -compile-compact-rule-table: - # Building the compact rule table tools may fail e.g. if boost is not available. - # We ignore this because the tools aren't used by default. - cd training/compact-rule-table ; \ - ./configure && $(MAKE) \ - || ( echo "WARNING: Building compact rule table tools failed."; \ - echo 'training/compact-rule-table/tools/compactify' >> ../../release-exclude ) - -compile-extract-ghkm: - # Building the GHKM rule extractor may fail e.g. if certain boost libraries are not available. - # We ignore this because the tool isn't used by default. - cd training/phrase-extract/extract-ghkm ; \ - if [ ! -f configure ]; then \ - autoreconf -i \ - || ( echo "WARNING: Bootstrapping extract-ghkm build environment failed."; \ - echo 'training/phrase-extract/extract-ghkm/tools/extract-ghkm' >> ../../release-exclude ); \ - fi; \ - ./configure && $(MAKE) \ - || ( echo "WARNING: Building extract-ghkm failed."; \ - echo 'training/phrase-extract/extract-ghkm/tools/extract-ghkm' >> ../../release-exclude ) - -clean: - pwd=`pwd`; \ - for subdir in $(SUBDIRS_CLEAN); do \ - $(MAKE) -C $$subdir clean || exit 1; \ - echo "### Compiler $$subdir"; \ - cd $$pwd; \ - done - ## All files that need compilation were compiled - - -### "MAIN" scripts are scripts that have a Philipp-like name, too -## That is for each script (listed below in MAIN_SCRIPTS), -## we create a date-stamped version in MAIN_SCRIPTS_TARGET_DIR - -MAIN_TRAINING_SCRIPTS_NAMES=filter-model-given-input.pl mert-moses-multi.pl mert-moses.pl train-model.perl clean-corpus-n.perl -# Make trick to add directory name to all of them: -MAIN_TRAINING_SCRIPTS=$(MAIN_TRAINING_SCRIPTS_NAMES:%=training/%) - -MAIN_GENERIC_SCRIPTS_NAMES= moses-parallel.pl -# Make trick to add directory name to all of them: -MAIN_GENERIC_SCRIPTS=$(MAIN_GENERIC_SCRIPTS_NAMES:%=generic/%) - -# the list of all scripts that should be released -MAIN_SCRIPTS= $(MAIN_TRAINING_SCRIPTS) $(MAIN_GENERIC_SCRIPTS) - - -release: - # Compile the parts - $(MAKE) all - @./check-dependencies.pl "$(HOME)" "$(TARGETDIR)" "$(RELEASEDIR)" "$(BINDIR)" - mkdir -p $(RELEASEDIR) - cat ./released-files \ - | grep -v -x -f release-exclude \ - | rsync -r --files-from - . $(RELEASEDIR)/ - sed 's#^my \$$BINDIR\s*=.*#my \$$BINDIR="$(BINDIR)";#' training/train-model.perl > $(RELEASEDIR)/training/train-model.perl - @echo "####### Do not forget to:" >> $(RELEASEDIR)/README - @echo " export SCRIPTS_ROOTDIR=$(RELEASEDIR)" >> $(RELEASEDIR)/README - ## Remember, only files listed in released-files are released!! - ## Don't forget to set your SCRIPTS_ROOTDIR with: - @echo " export SCRIPTS_ROOTDIR=$(RELEASEDIR)" - -generate_wrappers: - ## And for each script, create/rewrite the daily release - export TARGET - @for s in $(MAIN_SCRIPTS); do \ - bn=`basename $$s`; \ - echo '#!/bin/bash' > $(MAIN_SCRIPTS_TARGET_DIR)/$$bn-$(DS) || exit 1; \ - echo "export SCRIPTS_ROOTDIR=$(RELEASEDIR); $(RELEASEDIR)/$$s "'"$$@"; exit $$?' >> $(MAIN_SCRIPTS_TARGET_DIR)/$$bn-$(DS) || exit 1; \ - chmod 775 $(MAIN_SCRIPTS_TARGET_DIR)/$$bn-$(DS); \ - done - - -MOSESRELEASE=$(TARGETDIR)/moses.$(DS) -## This is a handy goal to release moses binary, too -releasemoses: - if [ -z "$(TARGETDIR)" ]; then \ - echo "Please specify a TARGETDIR." ; \ - echo " For custom releases issue: "; \ - echo " TARGETDIR=$(HOME)/releases make releasemoses"; \ - echo " For official releases: "; \ - echo " TARGETDIR=/export/ws06osmt make releasemoses"; \ - exit 1; \ - fi - if [ -e $(MOSESRELEASE) ]; then echo "Moses release exists! Not touching it! $(MOSESRELEASE)"; exit 1; fi - if [ ! -e ../moses-cmd/src/moses ]; then echo "Moses (../moses-cmd/src/moses) does not exist, nothing to release"; ecit 1; fi - if file ../moses-cmd/src/moses | grep -q 'dynamicall' ; then echo "Moses (../moses-cmd/src/moses) is dynamically linked, not releasing."; ecit 1; fi - cp ../moses-cmd/src/moses $(MOSESRELEASE) - ## Your current version of moses: - @echo " $(MOSESRELEASE)" - - -## This goal lists all files you might have wanted to release -# but forgot to mention in released-files -missed: - ### These might be intended for release - find . -type f \ - | grep -v '/CVS/' \ - | grep -v /tests/ \ - | sed 's/^\.\///' \ - | grep -F -x -v -f released-files - - -### Tests, applicable only at JHU environment due to data dependencies -export WORKSPACE=$(shell pwd)/../ - -.PHONY: tests -tests: - export SCRIPTS_ROOTDIR=`pwd`; \ - cd tests; \ - ts=`date '+%Y%m%d-%H%M%N'`; \ - for test in *.test; do \ - mkdir $$test.$$ts; \ - cd $$test.$$ts; \ - echo "Running $$test in tests/$$test.$$ts"; \ - ../$$test > log 2>&1 || exit 1; \ - cd ..; \ - done - ## All tests passed - -## Run just one test in the background -tests/%.test.run: tests/%.test - export SCRIPTS_ROOTDIR=`pwd`; \ - ts=`date '+%Y%m%d-%H%M%N'`; \ - cd tests; \ - test=$*.test; \ - mkdir $$test.$$ts; \ - cd $$test.$$ts; \ - echo "Running $$test in tests/$$test.$$ts"; \ - ( nohup ../$$test > log 2>&1 & ) || exit 1; \ - echo "Observe tests/$$test.$$ts/log"; \ - cd .. - -## Run just one test in the foreground -tests/%.test.runfg: tests/%.test - export SCRIPTS_ROOTDIR=`pwd`; \ - ts=`date '+%Y%m%d-%H%M%N'`; \ - cd tests; \ - test=$*.test; \ - mkdir $$test.$$ts; \ - cd $$test.$$ts; \ - echo "Running $$test in tests/$$test.$$ts"; \ - ../$$test 2>&1 | tee log ; \ - echo "Log saved to tests/$$test.$$ts/log"; \ - cd .. - diff --git a/scripts/README b/scripts/README index 609352bcc..35dac9dd0 100644 --- a/scripts/README +++ b/scripts/README @@ -8,7 +8,7 @@ This directory should contain all multi-purpose scripts for: - lib ... perl modules used by various scripts -The Makefile then takes care of proper 'release' from your CVS directory to +The Jamfile then takes care of proper 'release' from your git directory to the shared directories. The released scripts should remain in the *same directory structure*. diff --git a/scripts/check-dependencies.pl b/scripts/check-dependencies.pl deleted file mode 100755 index e93520c8d..000000000 --- a/scripts/check-dependencies.pl +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/perl - -# $Id$ - -my ($home, $target_dir, $release_dir, $bin_dir) = @ARGV; - -#print "HOME: $home\nTARGET_DIR: $target_dir\nRELEASE_DIR: $release_dir\n"; - -if ($target_dir eq '' || -z $target_dir) { - print <<EOT; -Please specify a TARGETDIR. - - For development releases you probably want the following: - TARGETDIR=$home/releases make release - - For shared environments, you will want to set TARGETDIR to - some appropriately common directory. - -EOT - exit 1; -} - -if (-e $release_dir) { - print "Targetdir exists! Not touching it! $release_dir"; - exit 1; -} - -unless (-x "$bin_dir/GIZA++" && -x "$bin_dir/snt2cooc.out" && -x "$bin_dir/mkcls" ) { - print <<EOT; -Please specify a BINDIR. - - The BINDIR directory must contain GIZA++, snt2cooc.out and mkcls executables. - These are available from http://www.fjoch.com/GIZA++.html and - http://www-i6.informatik.rwth-aachen.de/Colleagues/och/software/mkcls.html . -EOT - exit 1; -} - - -exit 0; - diff --git a/scripts/released-files b/scripts/released-files deleted file mode 100644 index 2e50c4a38..000000000 --- a/scripts/released-files +++ /dev/null @@ -1,107 +0,0 @@ -analysis/README -analysis/sentence-by-sentence.pl -ems/experiment.machines -ems/experiment.meta -ems/experiment.perl -ems/example/config.basic -ems/example/config.factored -ems/example/config.hierarchical -ems/example/config.syntax -ems/example/config.toy -ems/example/data/nc-5k.en -ems/example/data/nc-5k.fr -ems/example/data/test-ref.en.sgm -ems/example/data/test-src.fr.sgm -ems/support/analysis.perl -ems/support/berkeley-process.sh -ems/support/berkeley-train.sh -ems/support/consolidate-training-data.perl -ems/support/generic-multicore-parallelizer.perl -ems/support/generic-parallelizer.perl -ems/support/input-from-sgm.perl -ems/support/interpolate-lm.perl -ems/support/reference-from-sgm.perl -ems/support/remove-segmenation-markup.perl -ems/support/report-experiment-scores.perl -ems/support/reuse-weights.perl -ems/support/run-command-on-multiple-refsets.perl -ems/support/wrap-xml.perl -ems/web/analysis.php -ems/web/analysis_diff.php -ems/web/comment.php -ems/web/diff.php -ems/web/index.php -ems/web/lib.php -ems/web/overview.php -ems/web/setup -ems/web/javascripts/builder.js -ems/web/javascripts/controls.js -ems/web/javascripts/dragdrop.js -ems/web/javascripts/effects.js -ems/web/javascripts/prototype.js -ems/web/javascripts/scriptaculous.js -ems/web/javascripts/slider.js -ems/web/javascripts/sound.js -ems/web/javascripts/unittest.js -generic/compound-splitter.perl -generic/extract-factors.pl -generic/lopar2pos.pl -generic/moses-parallel.pl -generic/mteval-v12.pl -generic/multi-bleu.perl -generic/qsub-wrapper.pl -Makefile -README -released-files -tokenizer/detokenizer.perl -tokenizer/tokenizer.perl -tokenizer/lowercase.perl -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ru -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ro -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.de -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.fr -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.el -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.is -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.pt -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.nl -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.it -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sl -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sk -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.pl -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.sv -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.es -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.en -tokenizer/nonbreaking_prefixes/nonbreaking_prefix.ca -training/absolutize_moses_model.pl -training/build-generation-table.perl -training/clean-corpus-n.perl -training/clone_moses_model.pl -training/compact-rule-table/tools/compactify -training/filter-model-given-input.pl -training/filter-rule-table.py -training/lexical-reordering/score -training/zmert-moses.pl -training/mert-moses.pl -training/mert-moses-multi.pl -training/phrase-extract/extract -training/phrase-extract/extract-ghkm/tools/extract-ghkm -training/phrase-extract/extract-rules -training/phrase-extract/score -training/phrase-extract/consolidate -training/postprocess-lopar.perl -training/reduce_combine.pl -training/combine_factors.pl -training/train-model.perl -training/symal/symal -training/symal/giza2bal.pl -training/wrappers/parse-de-bitpar.perl -training/wrappers/parse-en-collins.perl -training/wrappers/make-factor-en-pos.mxpost.perl -training/wrappers/make-factor-pos.tree-tagger.perl -training/wrappers/make-factor-stem.perl -recaser/train-recaser.perl -recaser/recase.perl -recaser/truecase.perl -recaser/detruecase.perl -recaser/train-truecaser.perl - diff --git a/scripts/training/Jamfile b/scripts/training/Jamfile index 0444af8d1..2de16d05c 100644 --- a/scripts/training/Jamfile +++ b/scripts/training/Jamfile @@ -7,3 +7,5 @@ make train-model.perl : train-model.perl.missing_bin_dir : @missing_bin_dir ; actions missing_bin_dir { sed 's#^my \$BINDIR\s*=.*#my\ \$BINDIR=\"$(BINDIR)\";#' $(>) >$(<) } + +install legacy : train-model.perl : <location>. ; |